<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>

<meta charset="utf-8">
<meta name="generator" content="quarto-1.8.23">

<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">


<title>The Book – The Little Book of Linear Algebra</title>
<style>
code{white-space: pre-wrap;}
span.smallcaps{font-variant: small-caps;}
div.columns{display: flex; gap: min(4vw, 1.5em);}
div.column{flex: auto; overflow-x: auto;}
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
ul.task-list{list-style: none;}
ul.task-list li input[type="checkbox"] {
  width: 0.8em;
  margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */ 
  vertical-align: middle;
}
</style>


<script src="../../site_libs/quarto-nav/quarto-nav.js"></script>
<script src="../../site_libs/quarto-nav/headroom.min.js"></script>
<script src="../../site_libs/clipboard/clipboard.min.js"></script>
<script src="../../site_libs/quarto-search/autocomplete.umd.js"></script>
<script src="../../site_libs/quarto-search/fuse.min.js"></script>
<script src="../../site_libs/quarto-search/quarto-search.js"></script>
<meta name="quarto:offset" content="../../">
<link href="../../books/en-US/lab.html" rel="next">
<link href="../../index.html" rel="prev">
<script src="../../site_libs/quarto-html/quarto.js" type="module"></script>
<script src="../../site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
<script src="../../site_libs/quarto-html/axe/axe-check.js" type="module"></script>
<script src="../../site_libs/quarto-html/popper.min.js"></script>
<script src="../../site_libs/quarto-html/tippy.umd.min.js"></script>
<script src="../../site_libs/quarto-html/anchor.min.js"></script>
<link href="../../site_libs/quarto-html/tippy.css" rel="stylesheet">
<link href="../../site_libs/quarto-html/quarto-syntax-highlighting-1fe81d0376b2c50856e68e651e390326.css" rel="stylesheet" id="quarto-text-highlighting-styles">
<script src="../../site_libs/bootstrap/bootstrap.min.js"></script>
<link href="../../site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
<link href="../../site_libs/bootstrap/bootstrap-27c261d06b905028a18691de25d09dde.min.css" rel="stylesheet" append-hash="true" id="quarto-bootstrap" data-mode="light">
<script id="quarto-search-options" type="application/json">{
  "location": "sidebar",
  "copy-button": false,
  "collapse-after": 3,
  "panel-placement": "start",
  "type": "textbox",
  "limit": 50,
  "keyboard-shortcut": [
    "f",
    "/",
    "s"
  ],
  "show-item-context": false,
  "language": {
    "search-no-results-text": "No results",
    "search-matching-documents-text": "matching documents",
    "search-copy-link-title": "Copy link to search",
    "search-hide-matches-text": "Hide additional matches",
    "search-more-match-text": "more match in this document",
    "search-more-matches-text": "more matches in this document",
    "search-clear-button-title": "Clear",
    "search-text-placeholder": "",
    "search-detached-cancel-button-title": "Cancel",
    "search-submit-button-title": "Submit",
    "search-label": "Search"
  }
}</script>

  <script src="https://cdnjs.cloudflare.com/polyfill/v3/polyfill.min.js?features=es6"></script>
  <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml-full.js" type="text/javascript"></script>

<script type="text/javascript">
const typesetMath = (el) => {
  if (window.MathJax) {
    // MathJax Typeset
    window.MathJax.typeset([el]);
  } else if (window.katex) {
    // KaTeX Render
    var mathElements = el.getElementsByClassName("math");
    var macros = [];
    for (var i = 0; i < mathElements.length; i++) {
      var texText = mathElements[i].firstChild;
      if (mathElements[i].tagName == "SPAN" && texText && texText.data) {
        window.katex.render(texText.data, mathElements[i], {
          displayMode: mathElements[i].classList.contains('display'),
          throwOnError: false,
          macros: macros,
          fleqn: false
        });
      }
    }
  }
}
window.Quarto = {
  typesetMath
};
</script>

</head>

<body class="nav-sidebar floating quarto-light">

<div id="quarto-search-results"></div>
  <header id="quarto-header" class="headroom fixed-top">
  <nav class="quarto-secondary-nav">
    <div class="container-fluid d-flex">
      <button type="button" class="quarto-btn-toggle btn" data-bs-toggle="collapse" role="button" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
        <i class="bi bi-layout-text-sidebar-reverse"></i>
      </button>
        <nav class="quarto-page-breadcrumbs" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="../../books/en-US/book.html"><span class="chapter-title">The Book</span></a></li></ol></nav>
        <a class="flex-grow-1" role="navigation" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">      
        </a>
      <button type="button" class="btn quarto-search-button" aria-label="Search" onclick="window.quartoOpenSearch();">
        <i class="bi bi-search"></i>
      </button>
    </div>
  </nav>
</header>
<!-- content -->
<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article">
<!-- sidebar -->
  <nav id="quarto-sidebar" class="sidebar collapse collapse-horizontal quarto-sidebar-collapse-item sidebar-navigation floating overflow-auto">
    <div class="pt-lg-2 mt-2 text-left sidebar-header sidebar-header-stacked">
      <a href="../../index.html" class="sidebar-logo-link">
      </a>
    <div class="sidebar-title mb-0 py-0">
      <a href="../../">The Little Book of Linear Algebra</a> 
    </div>
      </div>
        <div class="mt-2 flex-shrink-0 align-items-center">
        <div class="sidebar-search">
        <div id="quarto-search" class="" title="Search"></div>
        </div>
        </div>
    <div class="sidebar-menu-container"> 
    <ul class="list-unstyled mt-1">
        <li class="sidebar-item">
  <div class="sidebar-item-container"> 
  <a href="../../index.html" class="sidebar-item-text sidebar-link"><span class="chapter-title">Content</span></a>
  </div>
</li>
        <li class="sidebar-item">
  <div class="sidebar-item-container"> 
  <a href="../../books/en-US/book.html" class="sidebar-item-text sidebar-link active"><span class="chapter-title">The Book</span></a>
  </div>
</li>
        <li class="sidebar-item">
  <div class="sidebar-item-container"> 
  <a href="../../books/en-US/lab.html" class="sidebar-item-text sidebar-link"><span class="chapter-title">The LAB</span></a>
  </div>
</li>
    </ul>
    </div>
    <nav id="TOC" role="doc-toc" class="toc-active">
    <h2 id="toc-title">Table of contents</h2>
   
  <ul>
  <li><a href="#overtune" id="toc-overtune" class="nav-link active" data-scroll-target="#overtune">Overtune</a></li>
  <li><a href="#chapter-1.-vectors-scalars-and-geometry" id="toc-chapter-1.-vectors-scalars-and-geometry" class="nav-link" data-scroll-target="#chapter-1.-vectors-scalars-and-geometry">Chapter 1. Vectors, scalars, and geometry</a>
  <ul class="collapse">
  <li><a href="#scalars-vectors-and-coordinate-systems" id="toc-scalars-vectors-and-coordinate-systems" class="nav-link" data-scroll-target="#scalars-vectors-and-coordinate-systems">1. Scalars, Vectors, and Coordinate Systems</a></li>
  <li><a href="#vector-notation-components-and-arrows" id="toc-vector-notation-components-and-arrows" class="nav-link" data-scroll-target="#vector-notation-components-and-arrows">2. Vector Notation, Components, and Arrows</a></li>
  <li><a href="#vector-addition-and-scalar-multiplication" id="toc-vector-addition-and-scalar-multiplication" class="nav-link" data-scroll-target="#vector-addition-and-scalar-multiplication">3. Vector Addition and Scalar Multiplication</a></li>
  <li><a href="#linear-combinations-and-span" id="toc-linear-combinations-and-span" class="nav-link" data-scroll-target="#linear-combinations-and-span">4. Linear Combinations and Span</a></li>
  <li><a href="#length-norm-and-distance" id="toc-length-norm-and-distance" class="nav-link" data-scroll-target="#length-norm-and-distance">5. Length (Norm) and Distance</a></li>
  <li><a href="#dot-product-algebraic-and-geometric-views" id="toc-dot-product-algebraic-and-geometric-views" class="nav-link" data-scroll-target="#dot-product-algebraic-and-geometric-views">6. Dot Product (Algebraic and Geometric Views)</a></li>
  <li><a href="#angles-between-vectors-and-cosine" id="toc-angles-between-vectors-and-cosine" class="nav-link" data-scroll-target="#angles-between-vectors-and-cosine">7. Angles Between Vectors and Cosine</a></li>
  <li><a href="#projections-and-decompositions" id="toc-projections-and-decompositions" class="nav-link" data-scroll-target="#projections-and-decompositions">8. Projections and Decompositions</a></li>
  <li><a href="#cauchyschwarz-and-triangle-inequalities" id="toc-cauchyschwarz-and-triangle-inequalities" class="nav-link" data-scroll-target="#cauchyschwarz-and-triangle-inequalities">9. Cauchy–Schwarz and Triangle Inequalities</a></li>
  <li><a href="#orthonormal-sets-in-mathbbr2-and-mathbbr3" id="toc-orthonormal-sets-in-mathbbr2-and-mathbbr3" class="nav-link" data-scroll-target="#orthonormal-sets-in-mathbbr2-and-mathbbr3">10. Orthonormal sets in <span class="math inline">\(\mathbb{R}^2\)</span> and <span class="math inline">\(\mathbb{R}^3\)</span></a></li>
  </ul></li>
  <li><a href="#chapter-2.-matrices-and-basic-operations" id="toc-chapter-2.-matrices-and-basic-operations" class="nav-link" data-scroll-target="#chapter-2.-matrices-and-basic-operations">Chapter 2. Matrices and basic operations</a>
  <ul class="collapse">
  <li><a href="#matrices-as-tables-and-as-machines" id="toc-matrices-as-tables-and-as-machines" class="nav-link" data-scroll-target="#matrices-as-tables-and-as-machines">11. Matrices as Tables and as Machines</a></li>
  <li><a href="#matrix-shapes-indexing-and-block-views" id="toc-matrix-shapes-indexing-and-block-views" class="nav-link" data-scroll-target="#matrix-shapes-indexing-and-block-views">12. Matrix Shapes, Indexing, and Block Views</a></li>
  <li><a href="#matrix-addition-and-scalar-multiplication" id="toc-matrix-addition-and-scalar-multiplication" class="nav-link" data-scroll-target="#matrix-addition-and-scalar-multiplication">13. Matrix Addition and Scalar Multiplication</a></li>
  <li><a href="#matrixvector-product-linear-combinations-of-columns" id="toc-matrixvector-product-linear-combinations-of-columns" class="nav-link" data-scroll-target="#matrixvector-product-linear-combinations-of-columns">14. Matrix–Vector Product (Linear Combinations of Columns)</a></li>
  <li><a href="#matrixmatrix-product-composition-of-linear-steps" id="toc-matrixmatrix-product-composition-of-linear-steps" class="nav-link" data-scroll-target="#matrixmatrix-product-composition-of-linear-steps">15. Matrix–Matrix Product (Composition of Linear Steps)</a></li>
  <li><a href="#identity-inverse-and-transpose" id="toc-identity-inverse-and-transpose" class="nav-link" data-scroll-target="#identity-inverse-and-transpose">16. Identity, Inverse, and Transpose</a></li>
  <li><a href="#symmetric-diagonal-triangular-and-permutation-matrices" id="toc-symmetric-diagonal-triangular-and-permutation-matrices" class="nav-link" data-scroll-target="#symmetric-diagonal-triangular-and-permutation-matrices">17. Symmetric, Diagonal, Triangular, and Permutation Matrices</a></li>
  <li><a href="#trace-and-basic-matrix-properties" id="toc-trace-and-basic-matrix-properties" class="nav-link" data-scroll-target="#trace-and-basic-matrix-properties">18. Trace and Basic Matrix Properties</a></li>
  <li><a href="#affine-transforms-and-homogeneous-coordinates" id="toc-affine-transforms-and-homogeneous-coordinates" class="nav-link" data-scroll-target="#affine-transforms-and-homogeneous-coordinates">19. Affine Transforms and Homogeneous Coordinates</a></li>
  <li><a href="#computing-with-matrices-cost-counts-and-simple-speedups" id="toc-computing-with-matrices-cost-counts-and-simple-speedups" class="nav-link" data-scroll-target="#computing-with-matrices-cost-counts-and-simple-speedups">20. Computing with Matrices (Cost Counts and Simple Speedups)</a></li>
  </ul></li>
  <li><a href="#chapter-3.-linear-systems-and-elimination" id="toc-chapter-3.-linear-systems-and-elimination" class="nav-link" data-scroll-target="#chapter-3.-linear-systems-and-elimination">Chapter 3. Linear Systems and Elimination</a>
  <ul class="collapse">
  <li><a href="#from-equations-to-matrices" id="toc-from-equations-to-matrices" class="nav-link" data-scroll-target="#from-equations-to-matrices">21. From Equations to Matrices</a></li>
  <li><a href="#row-operations" id="toc-row-operations" class="nav-link" data-scroll-target="#row-operations">22. Row Operations</a></li>
  <li><a href="#row-echelon-and-reduced-row-echelon-forms" id="toc-row-echelon-and-reduced-row-echelon-forms" class="nav-link" data-scroll-target="#row-echelon-and-reduced-row-echelon-forms">23. Row-Echelon and Reduced Row-Echelon Forms</a></li>
  <li><a href="#pivots-free-variables-and-leading-ones" id="toc-pivots-free-variables-and-leading-ones" class="nav-link" data-scroll-target="#pivots-free-variables-and-leading-ones">24. Pivots, Free Variables, and Leading Ones</a></li>
  <li><a href="#solving-consistent-systems" id="toc-solving-consistent-systems" class="nav-link" data-scroll-target="#solving-consistent-systems">25. Solving Consistent Systems</a></li>
  <li><a href="#detecting-inconsistency" id="toc-detecting-inconsistency" class="nav-link" data-scroll-target="#detecting-inconsistency">26. Detecting Inconsistency</a></li>
  <li><a href="#gaussian-elimination-by-hand" id="toc-gaussian-elimination-by-hand" class="nav-link" data-scroll-target="#gaussian-elimination-by-hand">27. Gaussian Elimination by Hand</a></li>
  <li><a href="#back-substitution-and-solution-sets" id="toc-back-substitution-and-solution-sets" class="nav-link" data-scroll-target="#back-substitution-and-solution-sets">28. Back Substitution and Solution Sets</a></li>
  <li><a href="#rank-and-its-first-meaning" id="toc-rank-and-its-first-meaning" class="nav-link" data-scroll-target="#rank-and-its-first-meaning">29. Rank and Its First Meaning</a></li>
  <li><a href="#lu-factorization" id="toc-lu-factorization" class="nav-link" data-scroll-target="#lu-factorization">30. LU Factorization</a></li>
  </ul></li>
  <li><a href="#chapter-4.-vector-spaces-and-subspaces" id="toc-chapter-4.-vector-spaces-and-subspaces" class="nav-link" data-scroll-target="#chapter-4.-vector-spaces-and-subspaces">Chapter 4. Vector spaces and subspaces</a>
  <ul class="collapse">
  <li><a href="#axioms-of-vector-spaces" id="toc-axioms-of-vector-spaces" class="nav-link" data-scroll-target="#axioms-of-vector-spaces">31. Axioms of Vector Spaces</a></li>
  <li><a href="#subspaces-column-space-and-null-space" id="toc-subspaces-column-space-and-null-space" class="nav-link" data-scroll-target="#subspaces-column-space-and-null-space">32. Subspaces, Column Space, and Null Space</a></li>
  <li><a href="#span-and-generating-sets" id="toc-span-and-generating-sets" class="nav-link" data-scroll-target="#span-and-generating-sets">33. Span and Generating Sets</a></li>
  <li><a href="#linear-independence-and-dependence" id="toc-linear-independence-and-dependence" class="nav-link" data-scroll-target="#linear-independence-and-dependence">34. Linear Independence and Dependence</a></li>
  <li><a href="#basis-and-coordinates" id="toc-basis-and-coordinates" class="nav-link" data-scroll-target="#basis-and-coordinates">35. Basis and Coordinates</a></li>
  <li><a href="#dimension-1" id="toc-dimension-1" class="nav-link" data-scroll-target="#dimension-1">36. Dimension</a></li>
  <li><a href="#ranknullity-theorem" id="toc-ranknullity-theorem" class="nav-link" data-scroll-target="#ranknullity-theorem">37. Rank–Nullity Theorem</a></li>
  <li><a href="#coordinates-relative-to-a-basis" id="toc-coordinates-relative-to-a-basis" class="nav-link" data-scroll-target="#coordinates-relative-to-a-basis">38. Coordinates Relative to a Basis</a></li>
  <li><a href="#change-of-basis-matrices" id="toc-change-of-basis-matrices" class="nav-link" data-scroll-target="#change-of-basis-matrices">39. Change-of-Basis Matrices</a></li>
  <li><a href="#affine-subspaces" id="toc-affine-subspaces" class="nav-link" data-scroll-target="#affine-subspaces">40. Affine Subspaces</a></li>
  </ul></li>
  <li><a href="#chapter-5.-linear-transformation-and-structure" id="toc-chapter-5.-linear-transformation-and-structure" class="nav-link" data-scroll-target="#chapter-5.-linear-transformation-and-structure">Chapter 5. Linear Transformation and Structure</a>
  <ul class="collapse">
  <li><a href="#linear-transformations" id="toc-linear-transformations" class="nav-link" data-scroll-target="#linear-transformations">41. Linear Transformations</a></li>
  <li><a href="#matrix-representation-of-a-linear-map" id="toc-matrix-representation-of-a-linear-map" class="nav-link" data-scroll-target="#matrix-representation-of-a-linear-map">42. Matrix Representation of a Linear Map</a></li>
  <li><a href="#kernel-and-image" id="toc-kernel-and-image" class="nav-link" data-scroll-target="#kernel-and-image">43. Kernel and Image</a></li>
  <li><a href="#invertibility-and-isomorphisms" id="toc-invertibility-and-isomorphisms" class="nav-link" data-scroll-target="#invertibility-and-isomorphisms">44. Invertibility and Isomorphisms</a></li>
  <li><a href="#composition-powers-and-iteration" id="toc-composition-powers-and-iteration" class="nav-link" data-scroll-target="#composition-powers-and-iteration">45. Composition, Powers, and Iteration</a></li>
  <li><a href="#similarity-and-conjugation" id="toc-similarity-and-conjugation" class="nav-link" data-scroll-target="#similarity-and-conjugation">46. Similarity and Conjugation</a></li>
  <li><a href="#projections-and-reflections" id="toc-projections-and-reflections" class="nav-link" data-scroll-target="#projections-and-reflections">47. Projections and Reflections</a></li>
  <li><a href="#rotations-and-shear" id="toc-rotations-and-shear" class="nav-link" data-scroll-target="#rotations-and-shear">48. Rotations and Shear</a></li>
  <li><a href="#rank-and-operator-viewpoint" id="toc-rank-and-operator-viewpoint" class="nav-link" data-scroll-target="#rank-and-operator-viewpoint">49. Rank and Operator Viewpoint</a></li>
  <li><a href="#block-matrices-and-block-maps" id="toc-block-matrices-and-block-maps" class="nav-link" data-scroll-target="#block-matrices-and-block-maps">50. Block Matrices and Block Maps</a></li>
  </ul></li>
  <li><a href="#chapter-6.-determinants-and-volume" id="toc-chapter-6.-determinants-and-volume" class="nav-link" data-scroll-target="#chapter-6.-determinants-and-volume">Chapter 6. Determinants and volume</a>
  <ul class="collapse">
  <li><a href="#areas-volumes-and-signed-scale-factors" id="toc-areas-volumes-and-signed-scale-factors" class="nav-link" data-scroll-target="#areas-volumes-and-signed-scale-factors">51. Areas, Volumes, and Signed Scale Factors</a></li>
  <li><a href="#determinant-via-linear-rules" id="toc-determinant-via-linear-rules" class="nav-link" data-scroll-target="#determinant-via-linear-rules">52. Determinant via Linear Rules</a></li>
  <li><a href="#determinant-and-row-operations" id="toc-determinant-and-row-operations" class="nav-link" data-scroll-target="#determinant-and-row-operations">53. Determinant and Row Operations</a></li>
  <li><a href="#triangular-matrices-and-product-of-diagonals" id="toc-triangular-matrices-and-product-of-diagonals" class="nav-link" data-scroll-target="#triangular-matrices-and-product-of-diagonals">54. Triangular Matrices and Product of Diagonals</a></li>
  <li><a href="#the-multiplicative-property-of-determinants-detab-detadetb" id="toc-the-multiplicative-property-of-determinants-detab-detadetb" class="nav-link" data-scroll-target="#the-multiplicative-property-of-determinants-detab-detadetb">55. The Multiplicative Property of Determinants: <span class="math inline">\(\det(AB) = \det(A)\det(B)\)</span></a></li>
  <li><a href="#invertibility-and-zero-determinant" id="toc-invertibility-and-zero-determinant" class="nav-link" data-scroll-target="#invertibility-and-zero-determinant">56. Invertibility and Zero Determinant</a></li>
  <li><a href="#cofactor-expansion" id="toc-cofactor-expansion" class="nav-link" data-scroll-target="#cofactor-expansion">57. Cofactor Expansion</a></li>
  <li><a href="#permutations-and-the-sign-of-the-determinant" id="toc-permutations-and-the-sign-of-the-determinant" class="nav-link" data-scroll-target="#permutations-and-the-sign-of-the-determinant">58. Permutations and the Sign of the Determinant</a></li>
  <li><a href="#cramers-rule" id="toc-cramers-rule" class="nav-link" data-scroll-target="#cramers-rule">59. Cramer’s Rule</a></li>
  <li><a href="#computing-determinants-in-practice" id="toc-computing-determinants-in-practice" class="nav-link" data-scroll-target="#computing-determinants-in-practice">60. Computing Determinants in Practice</a></li>
  </ul></li>
  <li><a href="#chapter-7.-eigenvalues-eigenvectors-and-dynamics" id="toc-chapter-7.-eigenvalues-eigenvectors-and-dynamics" class="nav-link" data-scroll-target="#chapter-7.-eigenvalues-eigenvectors-and-dynamics">Chapter 7. Eigenvalues, eigenvectors, and dynamics</a>
  <ul class="collapse">
  <li><a href="#eigenvalues-and-eigenvectors" id="toc-eigenvalues-and-eigenvectors" class="nav-link" data-scroll-target="#eigenvalues-and-eigenvectors">61. Eigenvalues and Eigenvectors</a></li>
  <li><a href="#the-characteristic-polynomial" id="toc-the-characteristic-polynomial" class="nav-link" data-scroll-target="#the-characteristic-polynomial">62. The Characteristic Polynomial</a></li>
  <li><a href="#algebraic-vs.-geometric-multiplicity" id="toc-algebraic-vs.-geometric-multiplicity" class="nav-link" data-scroll-target="#algebraic-vs.-geometric-multiplicity">63. Algebraic vs.&nbsp;Geometric Multiplicity</a></li>
  <li><a href="#diagonalization" id="toc-diagonalization" class="nav-link" data-scroll-target="#diagonalization">64. Diagonalization</a></li>
  <li><a href="#powers-of-a-matrix" id="toc-powers-of-a-matrix" class="nav-link" data-scroll-target="#powers-of-a-matrix">65. Powers of a Matrix</a></li>
  <li><a href="#real-vs.-complex-spectra" id="toc-real-vs.-complex-spectra" class="nav-link" data-scroll-target="#real-vs.-complex-spectra">66. Real vs.&nbsp;Complex Spectra</a></li>
  <li><a href="#defective-matrices-and-jordan-form-a-glimpse" id="toc-defective-matrices-and-jordan-form-a-glimpse" class="nav-link" data-scroll-target="#defective-matrices-and-jordan-form-a-glimpse">67. Defective Matrices and Jordan Form (a Glimpse)</a></li>
  <li><a href="#stability-and-spectral-radius" id="toc-stability-and-spectral-radius" class="nav-link" data-scroll-target="#stability-and-spectral-radius">68. Stability and Spectral Radius</a></li>
  <li><a href="#markov-chains-and-steady-states" id="toc-markov-chains-and-steady-states" class="nav-link" data-scroll-target="#markov-chains-and-steady-states">69. Markov Chains and Steady States</a></li>
  <li><a href="#linear-differential-systems" id="toc-linear-differential-systems" class="nav-link" data-scroll-target="#linear-differential-systems">70. Linear Differential Systems</a></li>
  </ul></li>
  <li><a href="#chapter-8.-orthogonality-least-squars-and-qr" id="toc-chapter-8.-orthogonality-least-squars-and-qr" class="nav-link" data-scroll-target="#chapter-8.-orthogonality-least-squars-and-qr">Chapter 8. Orthogonality, least squars, and QR</a>
  <ul class="collapse">
  <li><a href="#inner-products-beyond-dot-product" id="toc-inner-products-beyond-dot-product" class="nav-link" data-scroll-target="#inner-products-beyond-dot-product">71. Inner Products Beyond Dot Product</a></li>
  <li><a href="#orthogonality-and-orthonormal-bases" id="toc-orthogonality-and-orthonormal-bases" class="nav-link" data-scroll-target="#orthogonality-and-orthonormal-bases">72. Orthogonality and Orthonormal Bases</a></li>
  <li><a href="#gramschmidt-process" id="toc-gramschmidt-process" class="nav-link" data-scroll-target="#gramschmidt-process">73. Gram–Schmidt Process</a></li>
  <li><a href="#projections-onto-subspaces" id="toc-projections-onto-subspaces" class="nav-link" data-scroll-target="#projections-onto-subspaces">74. Projections onto Subspaces</a></li>
  <li><a href="#orthogonal-decomposition-theorem" id="toc-orthogonal-decomposition-theorem" class="nav-link" data-scroll-target="#orthogonal-decomposition-theorem">75. Orthogonal Decomposition Theorem</a></li>
  <li><a href="#orthogonal-projections-and-least-squares" id="toc-orthogonal-projections-and-least-squares" class="nav-link" data-scroll-target="#orthogonal-projections-and-least-squares">76. Orthogonal Projections and Least Squares</a></li>
  <li><a href="#qr-decomposition" id="toc-qr-decomposition" class="nav-link" data-scroll-target="#qr-decomposition">77. QR Decomposition</a></li>
  <li><a href="#orthogonal-matrices" id="toc-orthogonal-matrices" class="nav-link" data-scroll-target="#orthogonal-matrices">78. Orthogonal Matrices</a></li>
  <li><a href="#fourier-viewpoint" id="toc-fourier-viewpoint" class="nav-link" data-scroll-target="#fourier-viewpoint">79. Fourier Viewpoint</a></li>
  <li><a href="#polynomial-and-multifeature-least-squares" id="toc-polynomial-and-multifeature-least-squares" class="nav-link" data-scroll-target="#polynomial-and-multifeature-least-squares">80. Polynomial and Multifeature Least Squares</a></li>
  </ul></li>
  <li><a href="#chapter-9.-svd-pca-and-conditioning" id="toc-chapter-9.-svd-pca-and-conditioning" class="nav-link" data-scroll-target="#chapter-9.-svd-pca-and-conditioning">Chapter 9. SVD, PCA, and conditioning</a>
  <ul class="collapse">
  <li><a href="#opening-7" id="toc-opening-7" class="nav-link" data-scroll-target="#opening-7">Opening</a></li>
  <li><a href="#singular-values-and-svd" id="toc-singular-values-and-svd" class="nav-link" data-scroll-target="#singular-values-and-svd">81. Singular Values and SVD</a></li>
  <li><a href="#geometry-of-svd" id="toc-geometry-of-svd" class="nav-link" data-scroll-target="#geometry-of-svd">82. Geometry of SVD</a></li>
  <li><a href="#relation-to-eigen-decompositions" id="toc-relation-to-eigen-decompositions" class="nav-link" data-scroll-target="#relation-to-eigen-decompositions">83. Relation to Eigen-Decompositions</a></li>
  <li><a href="#low-rank-approximation-best-small-models" id="toc-low-rank-approximation-best-small-models" class="nav-link" data-scroll-target="#low-rank-approximation-best-small-models">84. Low-Rank Approximation (Best Small Models)</a></li>
  <li><a href="#principal-component-analysis-variance-and-directions" id="toc-principal-component-analysis-variance-and-directions" class="nav-link" data-scroll-target="#principal-component-analysis-variance-and-directions">85. Principal Component Analysis (Variance and Directions)</a></li>
  <li><a href="#pseudoinverse-moorepenrose-and-solving-ill-posed-systems" id="toc-pseudoinverse-moorepenrose-and-solving-ill-posed-systems" class="nav-link" data-scroll-target="#pseudoinverse-moorepenrose-and-solving-ill-posed-systems">86. Pseudoinverse (Moore–Penrose) and Solving Ill-Posed Systems</a></li>
  <li><a href="#conditioning-and-sensitivity-how-errors-amplify" id="toc-conditioning-and-sensitivity-how-errors-amplify" class="nav-link" data-scroll-target="#conditioning-and-sensitivity-how-errors-amplify">87. Conditioning and Sensitivity (How Errors Amplify)</a></li>
  <li><a href="#matrix-norms-and-singular-values-measuring-size-properly" id="toc-matrix-norms-and-singular-values-measuring-size-properly" class="nav-link" data-scroll-target="#matrix-norms-and-singular-values-measuring-size-properly">88. Matrix Norms and Singular Values (Measuring Size Properly)</a></li>
  <li><a href="#regularization-ridgetikhonov-to-tame-instability" id="toc-regularization-ridgetikhonov-to-tame-instability" class="nav-link" data-scroll-target="#regularization-ridgetikhonov-to-tame-instability">89. Regularization (Ridge/Tikhonov to Tame Instability)</a></li>
  <li><a href="#rank-revealing-qr-and-practical-diagnostics-what-rank-really-is" id="toc-rank-revealing-qr-and-practical-diagnostics-what-rank-really-is" class="nav-link" data-scroll-target="#rank-revealing-qr-and-practical-diagnostics-what-rank-really-is">90. Rank-Revealing QR and Practical Diagnostics (What Rank Really Is)</a></li>
  </ul></li>
  <li><a href="#chapter-10.-applications-and-computation" id="toc-chapter-10.-applications-and-computation" class="nav-link" data-scroll-target="#chapter-10.-applications-and-computation">Chapter 10. Applications and computation</a>
  <ul class="collapse">
  <li><a href="#d3d-geometry-pipelines-cameras-rotations-and-transforms" id="toc-d3d-geometry-pipelines-cameras-rotations-and-transforms" class="nav-link" data-scroll-target="#d3d-geometry-pipelines-cameras-rotations-and-transforms">91. 2D/3D Geometry Pipelines (Cameras, Rotations, and Transforms)</a></li>
  <li><a href="#computer-graphics-and-robotics-homogeneous-tricks-in-action" id="toc-computer-graphics-and-robotics-homogeneous-tricks-in-action" class="nav-link" data-scroll-target="#computer-graphics-and-robotics-homogeneous-tricks-in-action">92. Computer Graphics and Robotics (Homogeneous Tricks in Action)</a></li>
  <li><a href="#graphs-adjacency-and-laplacians-networks-via-matrices" id="toc-graphs-adjacency-and-laplacians-networks-via-matrices" class="nav-link" data-scroll-target="#graphs-adjacency-and-laplacians-networks-via-matrices">93. Graphs, Adjacency, and Laplacians (Networks via Matrices)</a></li>
  <li><a href="#data-preprocessing-as-linear-operations-centering-whitening-scaling" id="toc-data-preprocessing-as-linear-operations-centering-whitening-scaling" class="nav-link" data-scroll-target="#data-preprocessing-as-linear-operations-centering-whitening-scaling">94. Data Preprocessing as Linear Operations (Centering, Whitening, Scaling)</a></li>
  <li><a href="#linear-regression-and-classification-from-model-to-matrix" id="toc-linear-regression-and-classification-from-model-to-matrix" class="nav-link" data-scroll-target="#linear-regression-and-classification-from-model-to-matrix">95. Linear Regression and Classification (From Model to Matrix)</a></li>
  <li><a href="#pca-in-practice-dimensionality-reduction-workflow" id="toc-pca-in-practice-dimensionality-reduction-workflow" class="nav-link" data-scroll-target="#pca-in-practice-dimensionality-reduction-workflow">96. PCA in Practice (Dimensionality Reduction Workflow)</a></li>
  <li><a href="#recommender-systems-and-low-rank-models-fill-the-missing-entries" id="toc-recommender-systems-and-low-rank-models-fill-the-missing-entries" class="nav-link" data-scroll-target="#recommender-systems-and-low-rank-models-fill-the-missing-entries">97. Recommender Systems and Low-Rank Models (Fill the Missing Entries)</a></li>
  <li><a href="#pagerank-and-random-walks-ranking-with-eigenvectors" id="toc-pagerank-and-random-walks-ranking-with-eigenvectors" class="nav-link" data-scroll-target="#pagerank-and-random-walks-ranking-with-eigenvectors">98. PageRank and Random Walks (Ranking with Eigenvectors)</a></li>
  <li><a href="#numerical-linear-algebra-essentials-floating-point-blaslapack" id="toc-numerical-linear-algebra-essentials-floating-point-blaslapack" class="nav-link" data-scroll-target="#numerical-linear-algebra-essentials-floating-point-blaslapack">99. Numerical Linear Algebra Essentials (Floating Point, BLAS/LAPACK)</a></li>
  <li><a href="#capstone-problem-sets-and-next-steps-a-roadmap-to-mastery" id="toc-capstone-problem-sets-and-next-steps-a-roadmap-to-mastery" class="nav-link" data-scroll-target="#capstone-problem-sets-and-next-steps-a-roadmap-to-mastery">100. Capstone Problem Sets and Next Steps (A Roadmap to Mastery)</a></li>
  <li><a href="#finale" id="toc-finale" class="nav-link" data-scroll-target="#finale">Finale</a></li>
  </ul></li>
  </ul>
</nav>
</nav>
<div id="quarto-sidebar-glass" class="quarto-sidebar-collapse-item" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item"></div>
<!-- margin-sidebar -->
    <div id="quarto-margin-sidebar" class="sidebar margin-sidebar zindex-bottom">
    </div>
<!-- main -->
<main class="content" id="quarto-document-content">

<header id="title-block-header" class="quarto-title-block default">
<div class="quarto-title">
<h1 class="title"><span class="chapter-title">The Book</span></h1>
</div>



<div class="quarto-title-meta">

    
  
    
  </div>
  


</header>


<section id="overtune" class="level3">
<h3 class="anchored" data-anchor-id="overtune">Overtune</h3>
<p><em>A soft opening, an invitation into the world of vectors and spaces, where each step begins a journey.</em></p>
<p><strong>1. Geometry’s Dawn</strong></p>
<pre><code>Lines cross in silence,
planes awaken with order,
numbers sketch the world.</code></pre>
<p><strong>2. Invitation to Learn</strong></p>
<pre><code>Steps begin with dots,
arrows stretch into new paths,
the journey unfolds.</code></pre>
<p><strong>3. Light and Shadow</strong></p>
<pre><code>Shadows fall on grids,
hidden shapes emerge in form,
clarity takes root.</code></pre>
<p><strong>4. The Seed of Structure</strong></p>
<pre><code>One point, then a line,
spaces blossom out from rules,
infinity grows.</code></pre>
<p><strong>5. Whisper of Algebra</strong></p>
<pre><code>Silent rules of space,
woven threads of thought align,
order sings through time.</code></pre>
<p><strong>6. Beginner’s Welcome</strong></p>
<pre><code>Empty page awaits,
axes cross like guiding hands,
first steps find their place.</code></pre>
<p><strong>7. Eternal Path</strong></p>
<pre><code>From vectors to stars,
equations trace destiny,
patterns guide our sight.</code></pre>
</section>
<section id="chapter-1.-vectors-scalars-and-geometry" class="level2">
<h2 class="anchored" data-anchor-id="chapter-1.-vectors-scalars-and-geometry">Chapter 1. Vectors, scalars, and geometry</h2>
<section id="opening" class="level4">
<h4 class="anchored" data-anchor-id="opening">Opening</h4>
<pre><code>Arrows in the air,
directions whisper softly—
the plane comes alive.</code></pre>
</section>
<section id="scalars-vectors-and-coordinate-systems" class="level3">
<h3 class="anchored" data-anchor-id="scalars-vectors-and-coordinate-systems">1. Scalars, Vectors, and Coordinate Systems</h3>
<p>When we begin learning linear algebra, everything starts with the simplest building blocks: scalars and vectors. A scalar is just a single number, like 3, –7, or π. It carries only magnitude and no direction. Scalars are what we use for counting, measuring length, or scaling other objects up and down. A vector, by contrast, is an ordered collection of numbers. You can picture it as an arrow pointing somewhere in space, or simply as a list like (2, 5) in 2D or (1, –3, 4) in 3D. Where scalars measure “how much,” vectors measure both “how much” and “which way.”</p>
<section id="coordinate-systems" class="level4">
<h4 class="anchored" data-anchor-id="coordinate-systems">Coordinate Systems</h4>
<p>To talk about vectors, we need a coordinate system. Imagine laying down two perpendicular axes on a sheet of paper: the x-axis (left to right) and the y-axis (up and down). Every point on the sheet can be described with two numbers: how far along the x-axis, and how far along the y-axis. This pair of numbers is a vector in 2D. Add a z-axis pointing up from the page, and you have 3D space. Each coordinate system gives us a way to describe vectors numerically, even though the underlying “space” is the same.</p>
</section>
<section id="visualizing-scalars-vs.-vectors" class="level4">
<h4 class="anchored" data-anchor-id="visualizing-scalars-vs.-vectors">Visualizing Scalars vs.&nbsp;Vectors</h4>
<ul>
<li>A scalar is like a single tick mark on a ruler.</li>
<li>A vector is like an arrow that starts at the origin (0, 0, …) and ends at the point defined by its components. For example, the vector (3, 4) in 2D points from the origin to the point 3 units along the x-axis and 4 units along the y-axis.</li>
</ul>
</section>
<section id="why-start-here" class="level4">
<h4 class="anchored" data-anchor-id="why-start-here">Why Start Here?</h4>
<p>Understanding the difference between scalars and vectors is the foundation for everything else in linear algebra. Every concept-matrices, linear transformations, eigenvalues-eventually reduces to how we manipulate vectors and scale them with scalars. Without this distinction, the rest of the subject would have no anchor.</p>
</section>
<section id="why-it-matters" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters">Why It Matters</h4>
<p>Nearly every field of science and engineering depends on this idea. Physics uses vectors for velocity, acceleration, and force. Computer graphics uses them to represent points, colors, and transformations. Data science treats entire datasets as high-dimensional vectors. By mastering scalars and vectors early, you unlock the language in which modern science and technology are written.</p>
</section>
<section id="try-it-yourself" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself">Try It Yourself</h4>
<ol type="1">
<li>Draw an x- and y-axis on a piece of paper. Plot the vector (2, 3).</li>
<li>Now draw the vector (–1, 4). Compare their directions and lengths.</li>
<li>Think: which of these two vectors points “more upward”? Which is “longer”?</li>
</ol>
<p>These simple experiments already give you intuition for the operations you’ll perform again and again in linear algebra.</p>
</section>
</section>
<section id="vector-notation-components-and-arrows" class="level3">
<h3 class="anchored" data-anchor-id="vector-notation-components-and-arrows">2. Vector Notation, Components, and Arrows</h3>
<p>Linear algebra gives us powerful ways to describe and manipulate vectors, but before we can do anything with them, we need a precise notation system. Notation is not just cosmetic-it tells us how to read, write, and think about vectors clearly and unambiguously. In this section, we’ll explore how vectors are written, how their components are represented, and how we can interpret them visually as arrows.</p>
<section id="writing-vectors" class="level4">
<h4 class="anchored" data-anchor-id="writing-vectors">Writing Vectors</h4>
<p>Vectors are usually denoted by lowercase letters in bold (like <span class="math inline">\(\mathbf{v}, \mathbf{w}, \mathbf{x}\)</span>)<br>
or with an arrow overhead (like <span class="math inline">\(\vec{v}\)</span>).</p>
<p>For instance, the vector <span class="math inline">\(\mathbf{v} = (2, 5)\)</span> is the same as <span class="math inline">\(\vec{v} = (2, 5)\)</span>.</p>
<p>The style depends on context: mathematicians often use bold, physicists often use arrows.<br>
In handwritten notes, people sometimes underline vectors (e.g., <span class="math inline">\(\underline{v}\)</span>) to avoid confusion with scalars.</p>
<p>The important thing is to distinguish vectors from scalars at a glance.</p>
</section>
<section id="components-of-a-vector" class="level4">
<h4 class="anchored" data-anchor-id="components-of-a-vector">Components of a Vector</h4>
<p>A vector in two dimensions has two components, written as <span class="math inline">\((x, y)\)</span>.<br>
In three dimensions, it has three components: <span class="math inline">\((x, y, z)\)</span>.<br>
More generally, an <span class="math inline">\(n\)</span>-dimensional vector has <span class="math inline">\(n\)</span> components: <span class="math inline">\((v_1, v_2, \ldots, v_n)\)</span>.<br>
Each component tells us how far the vector extends along one axis of the coordinate system.</p>
<p>For example:</p>
<ul>
<li><span class="math inline">\(\mathbf{v} = (3, 4)\)</span> means the vector extends 3 units along the <span class="math inline">\(x\)</span>-axis and 4 units along the <span class="math inline">\(y\)</span>-axis.</li>
<li><span class="math inline">\(\mathbf{w} = (-2, 0, 5)\)</span> means the vector extends <span class="math inline">\(-2\)</span> units along the <span class="math inline">\(x\)</span>-axis, <span class="math inline">\(0\)</span> along the <span class="math inline">\(y\)</span>-axis, and 5 along the <span class="math inline">\(z\)</span>-axis.</li>
</ul>
<p>We often refer to the <span class="math inline">\(i\)</span>-th component of a vector <span class="math inline">\(\mathbf{v}\)</span> as <span class="math inline">\(v_i\)</span>.<br>
So, for <span class="math inline">\(\mathbf{v} = (3, 4, 5)\)</span>, we have <span class="math inline">\(v_1 = 3\)</span>, <span class="math inline">\(v_2 = 4\)</span>, <span class="math inline">\(v_3 = 5\)</span>.</p>
</section>
<section id="column-vs.-row-vectors" class="level4">
<h4 class="anchored" data-anchor-id="column-vs.-row-vectors">Column vs.&nbsp;Row Vectors</h4>
<p>Vectors can be written in two common ways:</p>
<ul>
<li><p>As a row vector: <span class="math inline">\((v_1, v_2, v_3)\)</span><br>
</p></li>
<li><p>As a column vector:</p>
<p><span class="math display">\[
\begin{bmatrix}
v_1 \\
v_2 \\
v_3
\end{bmatrix}
\]</span></p></li>
</ul>
<p>Both represent the same abstract object.<br>
Row vectors are convenient for quick writing, while column vectors are essential when we start multiplying by matrices, because the dimensions must align.</p>
</section>
<section id="vectors-as-arrows" class="level4">
<h4 class="anchored" data-anchor-id="vectors-as-arrows">Vectors as Arrows</h4>
<p>The most intuitive way to picture a vector is as an arrow:</p>
<ul>
<li>It starts at the origin (0, 0, …).</li>
<li>It ends at the point given by its components.</li>
</ul>
<p>For example, the vector (2, 3) in 2D is drawn as an arrow from (0, 0) to (2, 3). The arrow has both direction (where it points) and magnitude (its length). This geometric picture makes abstract algebraic manipulations much easier to grasp.</p>
</section>
<section id="position-vectors-vs.-free-vectors" class="level4">
<h4 class="anchored" data-anchor-id="position-vectors-vs.-free-vectors">Position Vectors vs.&nbsp;Free Vectors</h4>
<p>There are two common interpretations of vectors:</p>
<ol type="1">
<li>Position vector - a vector that points from the origin to a specific point in space. Example: (2, 3) is the position vector for the point (2, 3).</li>
<li>Free vector - an arrow with length and direction, but not tied to a specific starting point. For instance, an arrow of length 5 pointing northeast can be drawn anywhere, but it still represents the same vector.</li>
</ol>
<p>In linear algebra, we often treat vectors as free vectors, because their meaning does not depend on where they are drawn.</p>
</section>
<section id="example-reading-a-vector" class="level4">
<h4 class="anchored" data-anchor-id="example-reading-a-vector">Example: Reading a Vector</h4>
<p>Suppose u = (–3, 2).</p>
<ul>
<li>The first component (–3) means move 3 units left along the x-axis.</li>
<li>The second component (2) means move 2 units up along the y-axis. So the arrow points to the point (–3, 2). Even without a diagram, the components tell us exactly what the arrow would look like.</li>
</ul>
</section>
<section id="why-it-matters-1" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-1">Why It Matters</h4>
<p>Clear notation is the backbone of linear algebra. Without it, equations quickly become unreadable, and intuition about direction and size is lost. The way we write vectors determines how easily we can connect the algebra (numbers and symbols) to the geometry (arrows and spaces). This dual perspective-symbolic and visual-is what makes linear algebra powerful and practical.</p>
</section>
<section id="try-it-yourself-1" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-1">Try It Yourself</h4>
<ol type="1">
<li>Write down the vector (4, –1). Draw it on graph paper.</li>
<li>Rewrite the same vector as a column vector.</li>
<li>Translate the vector (4, –1) by moving its starting point to (2, 3) instead of the origin. Notice that the arrow looks the same-it just starts elsewhere.</li>
<li>For a harder challenge: draw the 3D vector (2, –1, 3). Even if you can’t draw perfectly in 3D, try to show each component along the x, y, and z axes.</li>
</ol>
<p>By practicing both the notation and the arrow picture, you’ll develop fluency in switching between abstract symbols and concrete visualizations. This skill will make every later concept in linear algebra far more intuitive.</p>
</section>
</section>
<section id="vector-addition-and-scalar-multiplication" class="level3">
<h3 class="anchored" data-anchor-id="vector-addition-and-scalar-multiplication">3. Vector Addition and Scalar Multiplication</h3>
<p>Once we know how to describe vectors with components and arrows, the next step is to learn how to combine them. Two fundamental operations form the backbone of linear algebra: adding vectors together and scaling vectors with numbers (scalars). These two moves, though simple, generate everything else we’ll build later. With them, we can describe motion, forces, data transformations, and more.</p>
<section id="vector-addition-in-coordinates" class="level4">
<h4 class="anchored" data-anchor-id="vector-addition-in-coordinates">Vector Addition in Coordinates</h4>
<p>Suppose we have two vectors in 2D:<br>
<span class="math inline">\(\mathbf{u} = (u_1, u_2), \quad \mathbf{v} = (v_1, v_2)\)</span>.</p>
<p>Their sum is defined as:<br>
<span class="math display">\[
\mathbf{u} + \mathbf{v} = (u_1 + v_1, \; u_2 + v_2).
\]</span></p>
<p>In words, you add corresponding components.<br>
This works in higher dimensions too:</p>
<p><span class="math display">\[
(u_1, u_2, \ldots, u_n) + (v_1, v_2, \ldots, v_n) = (u_1 + v_1, \; u_2 + v_2, \; \ldots, \; u_n + v_n).
\]</span></p>
<p>Example: <span class="math display">\[
(2, 3) + (-1, 4) = (2 - 1, \; 3 + 4) = (1, 7).
\]</span></p>
</section>
<section id="vector-addition-as-geometry" class="level4">
<h4 class="anchored" data-anchor-id="vector-addition-as-geometry">Vector Addition as Geometry</h4>
<p>The geometric picture is even more illuminating. If you draw vector u as an arrow, then place the tail of v at the head of u, the arrow from the start of u to the head of v is u + v. This is called the tip-to-tail rule. The parallelogram rule is another visualization: place u and v tail-to-tail, form a parallelogram, and the diagonal is their sum.</p>
<p>Example: u = (3, 1), v = (2, 2). Draw both from the origin. Their sum (5, 3) is exactly the diagonal of the parallelogram they span.</p>
</section>
<section id="scalar-multiplication-in-coordinates" class="level4">
<h4 class="anchored" data-anchor-id="scalar-multiplication-in-coordinates">Scalar Multiplication in Coordinates</h4>
<p>Scalars stretch or shrink vectors.<br>
If <span class="math inline">\(\mathbf{u} = (u_1, u_2, \ldots, u_n)\)</span> and <span class="math inline">\(c\)</span> is a scalar, then:</p>
<p><span class="math display">\[
c \cdot \mathbf{u} = (c \cdot u_1, \; c \cdot u_2, \; \ldots, \; c \cdot u_n).
\]</span></p>
<p>Example:</p>
<p><span class="math display">\[
2 \cdot (3, 4) = (6, 8).
\]</span></p>
<p><span class="math display">\[
(-1) \cdot (3, 4) = (-3, -4).
\]</span></p>
<p>Multiplying by a positive scalar stretches or compresses the arrow while keeping the direction the same. Multiplying by a negative scalar flips the arrow to point the opposite way.</p>
</section>
<section id="scalar-multiplication-as-geometry" class="level4">
<h4 class="anchored" data-anchor-id="scalar-multiplication-as-geometry">Scalar Multiplication as Geometry</h4>
<p>Imagine the vector (1, 2). Draw it on graph paper: it goes right 1, up 2. Now double it: (2, 4). The arrow points in the same direction but is twice as long. Halve it: (0.5, 1). It’s the same direction but shorter. Negate it: (–1, –2). Now the arrow points backward.</p>
<p>This geometric picture explains why we call these numbers “scalars”: they scale the vector.</p>
</section>
<section id="combining-both-linear-combinations" class="level4">
<h4 class="anchored" data-anchor-id="combining-both-linear-combinations">Combining Both: Linear Combinations</h4>
<p>Vector addition and scalar multiplication are not just separate tricks-they combine to form the heart of linear algebra: linear combinations.</p>
<p>A linear combination of vectors <span class="math inline">\(u\)</span> and <span class="math inline">\(v\)</span> is any vector of the form<br>
<span class="math inline">\(a \cdot u + b \cdot v\)</span>, where <span class="math inline">\(a\)</span> and <span class="math inline">\(b\)</span> are scalars.</p>
<p>Example:<br>
If <span class="math inline">\(u = (1, 0)\)</span> and <span class="math inline">\(v = (0, 1)\)</span>, then<br>
<span class="math inline">\(3 \cdot u + 2 \cdot v = (3, 2)\)</span>.</p>
<p>This shows how any point on the grid can be reached by scaling and adding these two basic vectors.<br>
That’s the essence of constructing spaces.</p>
</section>
<section id="algebraic-properties" class="level4">
<h4 class="anchored" data-anchor-id="algebraic-properties">Algebraic Properties</h4>
<p>Vector addition and scalar multiplication obey rules that mirror arithmetic with numbers:</p>
<ul>
<li>Commutativity: <span class="math inline">\(u + v = v + u\)</span><br>
</li>
<li>Associativity: <span class="math inline">\((u + v) + w = u + (v + w)\)</span><br>
</li>
<li>Distributivity over scalars: <span class="math inline">\(c \cdot (u + v) = c \cdot u + c \cdot v\)</span><br>
</li>
<li>Distributivity over numbers: <span class="math inline">\((a + b) \cdot u = a \cdot u + b \cdot u\)</span></li>
</ul>
<p>These rules are not trivial bookkeeping - they guarantee that linear algebra behaves predictably,<br>
which is why it works as the language of science.</p>
</section>
<section id="why-it-matters-2" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-2">Why It Matters</h4>
<p>With only these two operations-addition and scaling-you can already describe lines, planes, and entire spaces. Any system that grows by combining influences, like physics, economics, or machine learning, is built on these simple rules. Later, when we define matrix multiplication, dot products, and eigenvalues, they all reduce to repeated patterns of adding and scaling vectors.</p>
</section>
<section id="try-it-yourself-2" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-2">Try It Yourself</h4>
<ol type="1">
<li>Add (2, 3) and (–1, 4). Draw the result on graph paper.</li>
<li>Multiply (1, –2) by 3, and then add (0, 5). What is the final vector?</li>
<li>For a deeper challenge: Let u = (1, 2) and v = (2, –1). Sketch all vectors of the form a·u + b·v for integer values of a, b between –2 and 2. Notice the grid of points you create-that’s the span of these two vectors.</li>
</ol>
<p>This simple practice shows you how combining two basic vectors through addition and scaling generates a whole structured space, the first glimpse of linear algebra’s real power.</p>
</section>
</section>
<section id="linear-combinations-and-span" class="level3">
<h3 class="anchored" data-anchor-id="linear-combinations-and-span">4. Linear Combinations and Span</h3>
<p>After learning to add vectors and scale them, the natural next question is: <em>what can we build from these two operations?</em> The answer is the concept of linear combinations, which leads directly to one of the most fundamental ideas in linear algebra: the span of a set of vectors. These ideas tell us not only what individual vectors can do, but how groups of vectors can shape entire spaces.</p>
<section id="what-is-a-linear-combination" class="level4">
<h4 class="anchored" data-anchor-id="what-is-a-linear-combination">What Is a Linear Combination?</h4>
<p>A linear combination is any vector formed by multiplying vectors with scalars and then adding the results together.</p>
<p>Formally, given vectors <span class="math inline">\(v_1, v_2, \ldots, v_k\)</span> and scalars <span class="math inline">\(a_1, a_2, \ldots, a_k\)</span>, a linear combination looks like:</p>
<p><span class="math display">\[
a_1 \cdot v_1 + a_2 \cdot v_2 + \cdots + a_k \cdot v_k.
\]</span></p>
<p>This is nothing more than repeated addition and scaling, but the idea is powerful because it describes how vectors combine to generate new ones.</p>
<p>Example:<br>
Let <span class="math inline">\(u = (1, 0)\)</span> and <span class="math inline">\(v = (0, 1)\)</span>. Then any linear combination <span class="math inline">\(a \cdot u + b \cdot v = (a, b)\)</span>.<br>
This shows that every point in the 2D plane can be expressed as a linear combination of these two simple vectors.</p>
</section>
<section id="geometric-meaning" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning">Geometric Meaning</h4>
<p>Linear combinations are about mixing directions and magnitudes. Each vector acts like a “directional ingredient,” and the scalars control how much of each ingredient you use.</p>
<ul>
<li>With one vector: You can only reach points on a single line through the origin.</li>
<li>With two non-parallel vectors in 2D: You can reach every point in the plane.</li>
<li>With three non-coplanar vectors in 3D: You can reach all of 3D space.</li>
</ul>
<p>This progression shows that the power of linear combinations depends not just on the vectors themselves but on how they relate to each other.</p>
</section>
<section id="the-span-of-a-set-of-vectors" class="level4">
<h4 class="anchored" data-anchor-id="the-span-of-a-set-of-vectors">The Span of a Set of Vectors</h4>
<p>The span of a set of vectors is the collection of all possible linear combinations of them.<br>
It answers the question: <em>“What space do these vectors generate?”</em></p>
<p>Notation:</p>
<p><span class="math display">\[
\text{Span}\{v_1, v_2, \ldots, v_k\} =
\{a_1 v_1 + a_2 v_2 + \cdots + a_k v_k \;|\; a_i \in \mathbb{R}\}.
\]</span></p>
<p>Examples:</p>
<ul>
<li><span class="math inline">\(\text{Span}\{(1, 0)\}\)</span> = all multiples of <span class="math inline">\((1, 0)\)</span>, which is the <span class="math inline">\(x\)</span>-axis.<br>
</li>
<li><span class="math inline">\(\text{Span}\{(1, 0), (0, 1)\}\)</span> = all of <span class="math inline">\(\mathbb{R}^2\)</span>, the entire plane.<br>
</li>
<li><span class="math inline">\(\text{Span}\{(1, 2), (2, 4)\}\)</span> = just the line through <span class="math inline">\((1, 2)\)</span>, because the second vector is a multiple of the first.</li>
</ul>
<p>So the span depends heavily on whether the vectors add new directions or just duplicate what’s already there.</p>
</section>
<section id="parallel-and-independent-vectors" class="level4">
<h4 class="anchored" data-anchor-id="parallel-and-independent-vectors">Parallel and Independent Vectors</h4>
<p>If vectors point in the same or opposite directions (one is a scalar multiple of another), then their span is just a line. They don’t add any new coverage of space. But if they point in different directions, they open up new dimensions. This leads to the critical idea of linear independence, which we’ll explore later: vectors are independent if none of them is a linear combination of the others.</p>
</section>
<section id="visualizing-span-in-different-dimensions" class="level4">
<h4 class="anchored" data-anchor-id="visualizing-span-in-different-dimensions">Visualizing Span in Different Dimensions</h4>
<ul>
<li><p>In 2D:</p>
<ul>
<li>One vector spans a line.</li>
<li>Two independent vectors span the whole plane.</li>
</ul></li>
<li><p>In 3D:</p>
<ul>
<li>One vector spans a line.</li>
<li>Two independent vectors span a plane.</li>
<li>Three independent vectors span all of 3D space.</li>
</ul></li>
<li><p>In higher dimensions: The same pattern continues. A set of k independent vectors spans a k-dimensional subspace inside the larger space.</p></li>
</ul>
</section>
<section id="algebraic-properties-1" class="level4">
<h4 class="anchored" data-anchor-id="algebraic-properties-1">Algebraic Properties</h4>
<ul>
<li>The span of vectors always includes the zero vector, because you can choose all scalars = 0.</li>
<li>The span is always a subspace, meaning it’s closed under addition and scalar multiplication. If you add two vectors in the span, the result stays in the span.</li>
<li>The span grows when you add new independent vectors, but not if the new vector is just a combination of the old ones.</li>
</ul>
</section>
<section id="why-it-matters-3" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-3">Why It Matters</h4>
<p>Linear combinations and span are the foundation for almost everything else in linear algebra:</p>
<ul>
<li>They define what it means for vectors to be independent or dependent.</li>
<li>They form the basis for solving linear systems (solutions are often described as spans).</li>
<li>They explain how dimensions arise in vector spaces.</li>
<li>They underpin practical methods like principal component analysis, where data is projected onto the span of a few important vectors.</li>
</ul>
<p>In short, the span tells us the “reach” of a set of vectors, and linear combinations are the mechanism to explore that reach.</p>
</section>
<section id="try-it-yourself-3" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-3">Try It Yourself</h4>
<ol type="1">
<li>Take vectors (1, 0) and (0, 1). Write down three different linear combinations and plot them. What shape do you notice?</li>
<li>Try vectors (1, 2) and (2, 4). Write down three different linear combinations. Plot them. What’s different from the previous case?</li>
<li>In 3D, consider (1, 0, 0) and (0, 1, 0). Describe their span. Add (0, 0, 1). How does the span change?</li>
<li>Challenge: Pick vectors (1, 2, 3) and (4, 5, 6). Do they span a plane or all of 3D space? How can you tell?</li>
</ol>
<p>By experimenting with simple examples, you’ll see clearly how the idea of span captures the richness or limitations of combining vectors.</p>
</section>
</section>
<section id="length-norm-and-distance" class="level3">
<h3 class="anchored" data-anchor-id="length-norm-and-distance">5. Length (Norm) and Distance</h3>
<p>So far, vectors have been arrows with direction and components. To compare them more meaningfully, we need ways to talk about how long they are and how far apart they are. These notions are formalized through the norm of a vector (its length) and the distance between vectors. These concepts tie together the algebra of components and the geometry of space.</p>
<section id="the-length-norm-of-a-vector" class="level4">
<h4 class="anchored" data-anchor-id="the-length-norm-of-a-vector">The Length (Norm) of a Vector</h4>
<p>The norm of a vector measures its magnitude, or how long the arrow is.<br>
For a vector <span class="math inline">\(v = (v_1, v_2, \ldots, v_n)\)</span> in <span class="math inline">\(n\)</span>-dimensional space, its norm is defined as:</p>
<p><span class="math display">\[
\|v\| = \sqrt{v_1^2 + v_2^2 + \cdots + v_n^2}.
\]</span></p>
<p>This formula comes directly from the Pythagorean theorem: the length of the hypotenuse equals the square root of the sum of squares of the legs.<br>
In 2D, this is the familiar distance formula between the origin and a point.</p>
<p>Examples:</p>
<ul>
<li><p>For <span class="math inline">\(v = (3, 4)\)</span>:<br>
<span class="math display">\[
\|v\| = \sqrt{3^2 + 4^2} = \sqrt{9 + 16} = 5.
\]</span></p></li>
<li><p>For <span class="math inline">\(w = (1, -2, 2)\)</span>:<br>
<span class="math display">\[
\|w\| = \sqrt{1^2 + (-2)^2 + 2^2} = \sqrt{1 + 4 + 4} = \sqrt{9} = 3.
\]</span></p></li>
</ul>
</section>
<section id="unit-vectors" class="level4">
<h4 class="anchored" data-anchor-id="unit-vectors">Unit Vectors</h4>
<p>A unit vector is a vector whose length is exactly 1.<br>
These are important because they capture direction without scaling.<br>
To create a unit vector from any nonzero vector, divide by its norm:</p>
<p><span class="math display">\[
u = \frac{v}{\|v\|}.
\]</span></p>
<p>Example:<br>
For <span class="math inline">\(v = (3, 4)\)</span>, the unit vector is</p>
<p><span class="math display">\[
u = \left(\tfrac{3}{5}, \tfrac{4}{5}\right).
\]</span></p>
<p>This points in the same direction as <span class="math inline">\((3, 4)\)</span> but has length 1.</p>
<p>Unit vectors are like pure directions.<br>
They’re especially useful for projections, defining coordinate systems, and normalizing data.</p>
</section>
<section id="distance-between-vectors" class="level4">
<h4 class="anchored" data-anchor-id="distance-between-vectors">Distance Between Vectors</h4>
<p>The distance between two vectors <span class="math inline">\(u\)</span> and <span class="math inline">\(v\)</span> is defined as the length of their difference:</p>
<p><span class="math display">\[
\text{dist}(u, v) = \|u - v\|.
\]</span></p>
<p>Example:<br>
Let <span class="math inline">\(u = (2, 1)\)</span> and <span class="math inline">\(v = (5, 5)\)</span>. Then</p>
<p><span class="math display">\[
u - v = (-3, -4).
\]</span></p>
<p>Its norm is</p>
<p><span class="math display">\[
\sqrt{(-3)^2 + (-4)^2} = \sqrt{9 + 16} = 5.
\]</span></p>
<p>So the distance is 5. This matches our intuition: the straight-line distance between points <span class="math inline">\((2, 1)\)</span> and <span class="math inline">\((5, 5)\)</span>.</p>
</section>
<section id="geometric-interpretation" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation">Geometric Interpretation</h4>
<ul>
<li>The norm tells you how far a point is from the origin.</li>
<li>The distance tells you how far two points are from each other.</li>
</ul>
<p>Both are computed with the same formula-the square root of sums of squares-but applied in slightly different contexts.</p>
</section>
<section id="different-kinds-of-norms" class="level4">
<h4 class="anchored" data-anchor-id="different-kinds-of-norms">Different Kinds of Norms</h4>
<p>The formula above defines the Euclidean norm (or <span class="math inline">\(\ell_2\)</span> norm), the most common one.<br>
But in linear algebra, other norms are also useful:</p>
<ul>
<li><p><span class="math inline">\(\ell_1\)</span> norm:<br>
<span class="math display">\[
\|v\|_1 = |v_1| + |v_2| + \cdots + |v_n|
\]</span><br>
(sum of absolute values).</p></li>
<li><p><span class="math inline">\(\ell_\infty\)</span> norm:<br>
<span class="math display">\[
\|v\|_\infty = \max(|v_1|, |v_2|, \ldots, |v_n|)
\]</span><br>
(largest component).</p></li>
</ul>
<p>These norms change the geometry of “length” and “distance.” For example, in the ℓ₁ norm, the unit circle is shaped like a diamond; in the ℓ∞ norm, it looks like a square.</p>
</section>
<section id="algebraic-properties-2" class="level4">
<h4 class="anchored" data-anchor-id="algebraic-properties-2">Algebraic Properties</h4>
<p>Norms and distances satisfy critical properties that make them consistent measures:</p>
<ul>
<li>Non-negativity: <span class="math inline">\(\|v\| \geq 0\)</span>, and <span class="math inline">\(\|v\| = 0\)</span> only if <span class="math inline">\(v = 0\)</span>.<br>
</li>
<li>Homogeneity: <span class="math inline">\(\|c \cdot v\| = |c| \, \|v\|\)</span> (scaling affects length predictably).<br>
</li>
<li>Triangle inequality: <span class="math inline">\(\|u + v\| \leq \|u\| + \|v\|\)</span> (the direct path is shortest).<br>
</li>
<li>Symmetry (for distance): <span class="math inline">\(\text{dist}(u, v) = \text{dist}(v, u)\)</span>.</li>
</ul>
<p>These properties are why norms and distances are robust tools across mathematics.</p>
</section>
<section id="why-it-matters-4" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-4">Why It Matters</h4>
<p>Understanding length and distance is the first step toward geometry in higher dimensions. These notions:</p>
<ul>
<li>Allow us to compare vectors quantitatively.</li>
<li>Form the basis of concepts like angles, orthogonality, and projections.</li>
<li>Underpin optimization problems (e.g., “find the closest vector” is central to machine learning).</li>
<li>Define the geometry of spaces, which changes dramatically depending on which norm you use.</li>
</ul>
</section>
<section id="try-it-yourself-4" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-4">Try It Yourself</h4>
<ol type="1">
<li>Compute the norm of (6, 8). Then divide by the norm to find its unit vector.</li>
<li>Find the distance between (1, 1, 1) and (4, 5, 6).</li>
<li>Compare the Euclidean and Manhattan (ℓ₁) distances between (0, 0) and (3, 4). Which one matches your intuition if you were walking along a city grid?</li>
<li>Challenge: For vectors u = (2, –1, 3) and v = (–2, 0, 1), compute ‖u – v‖. Then explain what this distance means geometrically.</li>
</ol>
<p>By working through these examples, you’ll see how norms and distances make abstract vectors feel as real as points and arrows you can measure in everyday life.</p>
</section>
</section>
<section id="dot-product-algebraic-and-geometric-views" class="level3">
<h3 class="anchored" data-anchor-id="dot-product-algebraic-and-geometric-views">6. Dot Product (Algebraic and Geometric Views)</h3>
<p>The dot product is one of the most fundamental operations in linear algebra. It looks like a simple formula, but it unlocks the ability to measure angles, detect orthogonality, project one vector onto another, and compute energy or work in physics. Understanding it requires seeing both the algebraic view (a formula on components) and the geometric view (a way to compare directions).</p>
<section id="algebraic-definition" class="level4">
<h4 class="anchored" data-anchor-id="algebraic-definition">Algebraic Definition</h4>
<p>For two vectors of the same dimension, <span class="math inline">\(u = (u_1, u_2, \ldots, u_n)\)</span> and <span class="math inline">\(v = (v_1, v_2, \ldots, v_n)\)</span>, the dot product is defined as:</p>
<p><span class="math display">\[
u \cdot v = u_1 v_1 + u_2 v_2 + \cdots + u_n v_n.
\]</span></p>
<p>This is simply multiplying corresponding components and summing the results.</p>
<p>Examples:</p>
<ul>
<li><span class="math inline">\((2, 3) \cdot (4, 5) = (2 \times 4) + (3 \times 5) = 8 + 15 = 23\)</span><br>
</li>
<li><span class="math inline">\((1, -2, 3) \cdot (0, 4, -1) = (1 \times 0) + (-2 \times 4) + (3 \times -1) = 0 - 8 - 3 = -11\)</span></li>
</ul>
<p>Notice that the dot product is always a scalar, not a vector.</p>
</section>
<section id="geometric-definition" class="level4">
<h4 class="anchored" data-anchor-id="geometric-definition">Geometric Definition</h4>
<p>The dot product can also be defined in terms of vector length and angle:</p>
<p><span class="math display">\[
u \cdot v = \|u\| \, \|v\| \cos(\theta),
\]</span></p>
<p>where <span class="math inline">\(\theta\)</span> is the angle between <span class="math inline">\(u\)</span> and <span class="math inline">\(v\)</span> (<span class="math inline">\(0^\circ \leq \theta \leq 180^\circ\)</span>).</p>
<p>This formula tells us:</p>
<ul>
<li>If the angle is acute (less than <span class="math inline">\(90^\circ\)</span>), <span class="math inline">\(\cos(\theta) &gt; 0\)</span>, so the dot product is positive.<br>
</li>
<li>If the angle is right (exactly <span class="math inline">\(90^\circ\)</span>), <span class="math inline">\(\cos(\theta) = 0\)</span>, so the dot product is 0.<br>
</li>
<li>If the angle is obtuse (greater than <span class="math inline">\(90^\circ\)</span>), <span class="math inline">\(\cos(\theta) &lt; 0\)</span>, so the dot product is negative.</li>
</ul>
<p>Thus, the sign of the dot product encodes directional alignment.</p>
</section>
<section id="connecting-the-two-definitions" class="level4">
<h4 class="anchored" data-anchor-id="connecting-the-two-definitions">Connecting the Two Definitions</h4>
<p>At first glance, the algebraic sum of products and the geometric length–angle formula seem unrelated. But they are equivalent. To see why, consider the law of cosines applied to a triangle formed by u, v, and u – v. Expanding both sides leads directly to the equivalence between the two formulas. This dual interpretation is what makes the dot product so powerful: it is both a computation rule and a geometric measurement.</p>
</section>
<section id="orthogonality" class="level4">
<h4 class="anchored" data-anchor-id="orthogonality">Orthogonality</h4>
<p>Two vectors are orthogonal (perpendicular) if and only if their dot product is zero:</p>
<p><span class="math display">\[
u \cdot v = 0 \;\;\Longleftrightarrow\;\; \theta = 90^\circ.
\]</span></p>
<p>This gives us an algebraic way to check for perpendicularity without drawing diagrams.</p>
<p>Example:<br>
<span class="math inline">\((2, 1) \cdot (-1, 2) = (2 \times -1) + (1 \times 2) = -2 + 2 = 0\)</span>,<br>
so the vectors are orthogonal.</p>
</section>
<section id="projections" class="level4">
<h4 class="anchored" data-anchor-id="projections">Projections</h4>
<p>The dot product also provides a way to project one vector onto another.<br>
The scalar projection of <span class="math inline">\(u\)</span> onto <span class="math inline">\(v\)</span> is:</p>
<p><span class="math display">\[
\text{proj}_{\text{scalar}}(u \text{ onto } v) = \frac{u \cdot v}{\|v\|}.
\]</span></p>
<p>The vector projection is then:</p>
<p><span class="math display">\[
\text{proj}_{\text{vector}}(u \text{ onto } v) = \frac{u \cdot v}{\|v\|^2} \, v.
\]</span></p>
<p>This allows us to decompose vectors into “parallel” and “perpendicular” components, which is central in geometry, physics, and data analysis.</p>
</section>
<section id="examples" class="level4">
<h4 class="anchored" data-anchor-id="examples">Examples</h4>
<ol type="1">
<li><p>Compute <span class="math inline">\(u = (3, 4)\)</span> and <span class="math inline">\(v = (4, 3)\)</span>.</p>
<ul>
<li>Dot product: <span class="math inline">\((3 \times 4) + (4 \times 3) = 12 + 12 = 24\)</span>.<br>
</li>
<li>Norms: <span class="math inline">\(\|u\| = 5\)</span>, <span class="math inline">\(\|v\| = 5\)</span>.<br>
</li>
<li><span class="math inline">\(\cos(\theta) = \tfrac{24}{5 \times 5} = \tfrac{24}{25} \approx 0.96\)</span>, so <span class="math inline">\(\theta \approx 16^\circ\)</span>.<br>
These vectors are nearly parallel.</li>
</ul></li>
<li><p>Compute <span class="math inline">\(u = (1, 2, -1)\)</span> and <span class="math inline">\(v = (2, -1, 1)\)</span>.</p>
<ul>
<li>Dot product: <span class="math inline">\((1 \times 2) + (2 \times -1) + (-1 \times 1) = 2 - 2 - 1 = -1\)</span>.<br>
</li>
<li>Norms: <span class="math inline">\(\|u\| = \sqrt{6}\)</span>, <span class="math inline">\(\|v\| = \sqrt{6}\)</span>.<br>
</li>
<li><span class="math inline">\(\cos(\theta) = \tfrac{-1}{\sqrt{6} \times \sqrt{6}} = -\tfrac{1}{6}\)</span>, so <span class="math inline">\(\theta \approx 99.6^\circ\)</span>.<br>
Slightly obtuse.</li>
</ul></li>
</ol>
</section>
<section id="physical-interpretation" class="level4">
<h4 class="anchored" data-anchor-id="physical-interpretation">Physical Interpretation</h4>
<p>In physics, the dot product computes work:</p>
<p><span class="math display">\[
\text{Work} = \text{Force} \cdot \text{Displacement}
           = \|\text{Force}\| \, \|\text{Displacement}\| \cos(\theta).
\]</span></p>
<p>Only the component of the force in the direction of motion contributes. If you push straight down on a box while trying to move it horizontally, the dot product is zero: no work is done in the direction of motion.</p>
</section>
<section id="algebraic-properties-3" class="level4">
<h4 class="anchored" data-anchor-id="algebraic-properties-3">Algebraic Properties</h4>
<ul>
<li>Commutative: <span class="math inline">\(u \cdot v = v \cdot u\)</span><br>
</li>
<li>Distributive: <span class="math inline">\(u \cdot (v + w) = u \cdot v + u \cdot w\)</span><br>
</li>
<li>Scalar compatibility: <span class="math inline">\((c \cdot u) \cdot v = c \,(u \cdot v)\)</span><br>
</li>
<li>Non-negativity: <span class="math inline">\(v \cdot v = \|v\|^2 \geq 0\)</span></li>
</ul>
<p>These guarantee that the dot product behaves consistently and meshes with the structure of vector spaces.</p>
</section>
<section id="why-it-matters-5" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-5">Why It Matters</h4>
<p>The dot product is the first bridge between algebra and geometry. It:</p>
<ul>
<li>Defines angles and orthogonality in higher dimensions.</li>
<li>Powers projections and decompositions, which underlie least squares, regression, and data fitting.</li>
<li>Appears in physics as energy, power, and work.</li>
<li>Serves as the kernel of many machine learning methods (e.g., similarity measures in high-dimensional spaces).</li>
</ul>
<p>Without the dot product, linear algebra would lack a way to connect numbers with geometry and meaning.</p>
</section>
<section id="try-it-yourself-5" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-5">Try It Yourself</h4>
<ol type="1">
<li>Compute (2, –1) · (–3, 4). Then find the angle between them.</li>
<li>Check if (1, 2, 3) and (2, 4, 6) are orthogonal. What does the dot product tell you?</li>
<li>Find the projection of (3, 1) onto (1, 2). Draw the original vector, the projection, and the perpendicular component.</li>
<li>In physics terms: Suppose a 10 N force is applied at 60° to the direction of motion, and the displacement is 5 m. How much work is done?</li>
</ol>
<p>These exercises reveal the dual power of the dot product: as a formula to compute and as a geometric tool to interpret.</p>
</section>
</section>
<section id="angles-between-vectors-and-cosine" class="level3">
<h3 class="anchored" data-anchor-id="angles-between-vectors-and-cosine">7. Angles Between Vectors and Cosine</h3>
<p>Having defined the dot product, we are now ready to measure angles between vectors. In everyday life, angles tell us how two lines or directions relate-whether they point the same way, are perpendicular, or are opposed. In linear algebra, the dot product and cosine function give us a precise, generalizable way to define angles in any dimension, not just in 2D or 3D. This section explores how we compute, interpret, and apply vector angles.</p>
<section id="the-definition-of-an-angle-between-vectors" class="level4">
<h4 class="anchored" data-anchor-id="the-definition-of-an-angle-between-vectors">The Definition of an Angle Between Vectors</h4>
<p>For two nonzero vectors <span class="math inline">\(u\)</span> and <span class="math inline">\(v\)</span>, the angle <span class="math inline">\(\theta\)</span> between them is defined by:</p>
<p><span class="math display">\[
\cos(\theta) = \frac{u \cdot v}{\|u\| \, \|v\|}.
\]</span></p>
<p>This formula comes directly from the geometric definition of the dot product.<br>
Rearranging gives:</p>
<p><span class="math display">\[
\theta = \arccos\!\left(\frac{u \cdot v}{\|u\| \, \|v\|}\right).
\]</span></p>
<p>Key points:</p>
<ul>
<li><span class="math inline">\(\theta\)</span> is always between <span class="math inline">\(0^\circ\)</span> and <span class="math inline">\(180^\circ\)</span> (or <span class="math inline">\(0\)</span> and <span class="math inline">\(\pi\)</span> radians).<br>
</li>
<li>The denominator normalizes the dot product by dividing by the product of lengths, so the result is dimensionless and always between <span class="math inline">\(-1\)</span> and <span class="math inline">\(1\)</span>.<br>
</li>
<li>The cosine value directly encodes alignment: positive, zero, or negative.</li>
</ul>
</section>
<section id="interpretation-of-cosine-values" class="level4">
<h4 class="anchored" data-anchor-id="interpretation-of-cosine-values">Interpretation of Cosine Values</h4>
<p>The cosine tells us about the directional relationship:</p>
<ul>
<li><span class="math inline">\(\cos(\theta) = 1 \;\;\Rightarrow\;\; \theta = 0^\circ\)</span> → vectors point in exactly the same direction.<br>
</li>
<li><span class="math inline">\(\cos(\theta) = 0 \;\;\Rightarrow\;\; \theta = 90^\circ\)</span> → vectors are orthogonal (perpendicular).<br>
</li>
<li><span class="math inline">\(\cos(\theta) = -1 \;\;\Rightarrow\;\; \theta = 180^\circ\)</span> → vectors point in exactly opposite directions.<br>
</li>
<li><span class="math inline">\(\cos(\theta) &gt; 0\)</span> → acute angle → vectors point more “together” than apart.<br>
</li>
<li><span class="math inline">\(\cos(\theta) &lt; 0\)</span> → obtuse angle → vectors point more “against” each other.</li>
</ul>
<p>Thus, the cosine compresses geometric alignment into a single number.</p>
</section>
<section id="examples-1" class="level4">
<h4 class="anchored" data-anchor-id="examples-1">Examples</h4>
<ol type="1">
<li><p><span class="math inline">\(u = (1, 0), \; v = (0, 1)\)</span></p>
<ul>
<li>Dot product: <span class="math inline">\(1 \times 0 + 0 \times 1 = 0\)</span><br>
</li>
<li>Norms: <span class="math inline">\(1\)</span> and <span class="math inline">\(1\)</span><br>
</li>
<li><span class="math inline">\(\cos(\theta) = 0 \;\Rightarrow\; \theta = 90^\circ\)</span><br>
The vectors are perpendicular, as expected.</li>
</ul></li>
<li><p><span class="math inline">\(u = (2, 3), \; v = (4, 6)\)</span></p>
<ul>
<li>Dot product: <span class="math inline">\((2 \times 4) + (3 \times 6) = 8 + 18 = 26\)</span><br>
</li>
<li>Norms: <span class="math inline">\(\sqrt{2^2 + 3^2} = \sqrt{13}\)</span>, and <span class="math inline">\(\sqrt{4^2 + 6^2} = \sqrt{52} = 2\sqrt{13}\)</span><br>
</li>
<li><span class="math inline">\(\cos(\theta) = \tfrac{26}{\sqrt{13} \cdot 2\sqrt{13}} = \tfrac{26}{26} = 1\)</span><br>
</li>
<li><span class="math inline">\(\theta = 0^\circ\)</span><br>
These vectors are multiples, so they align perfectly.</li>
</ul></li>
<li><p><span class="math inline">\(u = (1, 1), \; v = (-1, 1)\)</span></p>
<ul>
<li>Dot product: <span class="math inline">\((1 \times -1) + (1 \times 1) = -1 + 1 = 0\)</span><br>
</li>
<li><span class="math inline">\(\cos(\theta) = 0 \;\Rightarrow\; \theta = 90^\circ\)</span><br>
The vectors are perpendicular, forming diagonals of a square.</li>
</ul></li>
</ol>
</section>
<section id="angles-in-higher-dimensions" class="level4">
<h4 class="anchored" data-anchor-id="angles-in-higher-dimensions">Angles in Higher Dimensions</h4>
<p>The beauty of the formula is that it works in any dimension.<br>
Even in <span class="math inline">\(\mathbb{R}^{100}\)</span> or higher, we can define the angle between two vectors using only their dot product and norms.</p>
<p>While we cannot visualize the geometry directly in high dimensions, the cosine formula still captures how aligned two directions are:</p>
<p><span class="math display">\[
\cos(\theta) = \frac{u \cdot v}{\|u\| \, \|v\|}.
\]</span></p>
<p>This is critical in machine learning, where data often lives in very high-dimensional spaces.</p>
</section>
<section id="cosine-similarity" class="level4">
<h4 class="anchored" data-anchor-id="cosine-similarity">Cosine Similarity</h4>
<p>The cosine of the angle between two vectors is often called cosine similarity. It is widely used in data analysis and machine learning to measure how similar two data vectors are, independent of their magnitude.</p>
<ul>
<li>In text mining, documents are turned into word-frequency vectors. Cosine similarity measures how “close in topic” two documents are, regardless of length.</li>
<li>In recommendation systems, cosine similarity compares user preference vectors to suggest similar users or items.</li>
</ul>
<p>This demonstrates how a geometric concept extends far beyond pure math.</p>
</section>
<section id="orthogonality-revisited" class="level4">
<h4 class="anchored" data-anchor-id="orthogonality-revisited">Orthogonality Revisited</h4>
<p>The angle formula reinforces the special role of orthogonality.<br>
If <span class="math inline">\(\cos(\theta) = 0\)</span>, then <span class="math inline">\(u \cdot v = 0\)</span>.</p>
<p>This means the dot product not only computes length but also serves as a direct test for perpendicularity.<br>
This algebraic shortcut is far easier than manually checking geometric right angles.</p>
</section>
<section id="angles-and-projections" class="level4">
<h4 class="anchored" data-anchor-id="angles-and-projections">Angles and Projections</h4>
<p>Angles are closely tied to projections.<br>
The length of the projection of <span class="math inline">\(u\)</span> onto <span class="math inline">\(v\)</span> is <span class="math inline">\(\|u\|\cos(\theta)\)</span>.</p>
<p>If the angle is small, the projection is large — most of <span class="math inline">\(u\)</span> lies in the direction of <span class="math inline">\(v\)</span>.<br>
If the angle is close to <span class="math inline">\(90^\circ\)</span>, the projection shrinks toward zero.</p>
<p>Thus, the cosine acts as a scaling factor between directions.</p>
</section>
<section id="why-it-matters-6" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-6">Why It Matters</h4>
<p>Angles between vectors provide:</p>
<ul>
<li>A way to generalize geometry beyond 2D/3D.</li>
<li>A measure of similarity in high-dimensional data.</li>
<li>The foundation for orthogonality, projections, and decomposition of spaces.</li>
<li>A tool for optimization: in gradient descent, for example, the angle between the gradient and step direction determines how effectively we reduce error.</li>
</ul>
<p>Without the ability to measure angles, we could not connect algebraic manipulations with geometric intuition or practical applications.</p>
</section>
<section id="try-it-yourself-6" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-6">Try It Yourself</h4>
<ol type="1">
<li>Compute the angle between (2, 1) and (1, –1). Interpret the result.</li>
<li>Find two vectors in 3D that form a 60° angle. Verify using the cosine formula.</li>
<li>Consider word vectors for “cat” and “dog” in a machine learning model. Why might cosine similarity be a better measure of similarity than Euclidean distance?</li>
<li>Challenge: In <span class="math inline">\(\mathbb{R}^3\)</span>, find a vector orthogonal to both (1, 2, 3) and (3, 2, 1). What angle does it make with each of them?</li>
</ol>
<p>By experimenting with these problems, you will see how angles provide the missing link between algebraic formulas and geometric meaning in linear algebra.</p>
</section>
</section>
<section id="projections-and-decompositions" class="level3">
<h3 class="anchored" data-anchor-id="projections-and-decompositions">8. Projections and Decompositions</h3>
<p>In earlier sections, we saw how the dot product measures alignment and how the cosine formula gives us angles between vectors. The next natural step is to use these tools to project one vector onto another. Projection is a way to “shadow” one vector onto the direction of another, splitting vectors into meaningful parts: one along a given direction and one perpendicular to it. This is the essence of decomposition, and it is everywhere in linear algebra, geometry, physics, and data science.</p>
<section id="scalar-projection" class="level4">
<h4 class="anchored" data-anchor-id="scalar-projection">Scalar Projection</h4>
<p>The scalar projection of a vector <span class="math inline">\(u\)</span> onto a vector <span class="math inline">\(v\)</span> measures how much of <span class="math inline">\(u\)</span> lies in the direction of <span class="math inline">\(v\)</span>. It is given by:</p>
<p><span class="math display">\[
\text{proj}_{\text{scalar}}(u \text{ onto } v) = \frac{u \cdot v}{\|v\|}.
\]</span></p>
<ul>
<li>If this value is positive, <span class="math inline">\(u\)</span> has a component pointing in the same direction as <span class="math inline">\(v\)</span>.<br>
</li>
<li>If it is negative, <span class="math inline">\(u\)</span> points partly in the opposite direction.<br>
</li>
<li>If it is zero, <span class="math inline">\(u\)</span> is completely perpendicular to <span class="math inline">\(v\)</span>.</li>
</ul>
<p>Example:<br>
<span class="math inline">\(u = (3, 4)\)</span>, <span class="math inline">\(v = (1, 0)\)</span>.<br>
Dot product: <span class="math inline">\((3 \times 1 + 4 \times 0) = 3\)</span>.<br>
<span class="math inline">\(\|v\| = 1\)</span>.<br>
So the scalar projection is <span class="math inline">\(3\)</span>. This tells us <span class="math inline">\(u\)</span> has a “shadow” of length <span class="math inline">\(3\)</span> on the <span class="math inline">\(x\)</span>-axis.</p>
</section>
<section id="vector-projection" class="level4">
<h4 class="anchored" data-anchor-id="vector-projection">Vector Projection</h4>
<p>The vector projection gives the actual arrow in the direction of <span class="math inline">\(v\)</span> that corresponds to this scalar amount:</p>
<p><span class="math display">\[
\text{proj}_{\text{vector}}(u \text{ onto } v) = \frac{u \cdot v}{\|v\|^2} \, v.
\]</span></p>
<p>This formula normalizes <span class="math inline">\(v\)</span> into a unit vector, then scales it by the scalar projection.<br>
The result is a new vector lying along <span class="math inline">\(v\)</span>, capturing exactly the “parallel” part of <span class="math inline">\(u\)</span>.</p>
<p>Example:<br>
<span class="math inline">\(u = (3, 4)\)</span>, <span class="math inline">\(v = (1, 2)\)</span></p>
<ul>
<li>Dot product: <span class="math inline">\(3 \times 1 + 4 \times 2 = 3 + 8 = 11\)</span><br>
</li>
<li>Norm squared of <span class="math inline">\(v\)</span>: <span class="math inline">\((1^2 + 2^2) = 5\)</span><br>
</li>
<li>Coefficient: <span class="math inline">\(11 / 5 = 2.2\)</span><br>
</li>
<li>Projection vector: <span class="math inline">\(2.2 \cdot (1, 2) = (2.2, 4.4)\)</span></li>
</ul>
<p>So the part of <span class="math inline">\((3, 4)\)</span> in the direction of <span class="math inline">\((1, 2)\)</span> is <span class="math inline">\((2.2, 4.4)\)</span>.</p>
</section>
<section id="perpendicular-component" class="level4">
<h4 class="anchored" data-anchor-id="perpendicular-component">Perpendicular Component</h4>
<p>Once we have the projection, we can find the perpendicular component (often called the rejection) simply by subtracting:</p>
<p><span class="math display">\[
u_{\perp} = u - \text{proj}_{\text{vector}}(u \text{ onto } v).
\]</span></p>
<p>This gives the part of <span class="math inline">\(u\)</span> that is entirely orthogonal to <span class="math inline">\(v\)</span>.</p>
<p>Example continued:<br>
<span class="math inline">\(u_{\perp} = (3, 4) - (2.2, 4.4) = (0.8, -0.4)\)</span></p>
<p>Check:<br>
<span class="math inline">\((0.8, -0.4) \cdot (1, 2) = 0.8 \times 1 + (-0.4) \times 2 = 0.8 - 0.8 = 0\)</span>.<br>
Indeed, orthogonal.</p>
</section>
<section id="geometric-picture" class="level4">
<h4 class="anchored" data-anchor-id="geometric-picture">Geometric Picture</h4>
<p>Projection is like dropping a perpendicular from one vector onto another. Imagine shining a light perpendicular to v: the shadow of u on the line spanned by v is the projection. This visualization explains why projections split vectors naturally into two pieces:</p>
<ul>
<li>Parallel part: Along the line of v.</li>
<li>Perpendicular part: Orthogonal to v, forming a right angle.</li>
</ul>
<p>Together, these two parts reconstruct the original vector exactly.</p>
</section>
<section id="decomposition-of-vectors" class="level4">
<h4 class="anchored" data-anchor-id="decomposition-of-vectors">Decomposition of Vectors</h4>
<p>Every vector <span class="math inline">\(u\)</span> can be decomposed relative to another vector <span class="math inline">\(v\)</span> into two parts:</p>
<p><span class="math display">\[
u = \text{proj}_{\text{vector}}(u \text{ onto } v) + \big(u - \text{proj}_{\text{vector}}(u \text{ onto } v)\big).
\]</span></p>
<p>This decomposition is unique and geometrically meaningful.<br>
It generalizes to subspaces: we can project onto entire planes or higher-dimensional spans, splitting a vector into a “within-subspace” part and a “perpendicular-to-subspace” part.</p>
</section>
<section id="applications" class="level4">
<h4 class="anchored" data-anchor-id="applications">Applications</h4>
<ol type="1">
<li><p>Physics (Work and Forces): Work is the projection of force onto displacement. Only the part of the force in the direction of motion contributes. Example: Pushing on a sled partly sideways wastes effort-the sideways component projects to zero.</p></li>
<li><p>Geometry and Engineering: Projections are used in CAD (computer-aided design) to flatten 3D objects onto 2D surfaces, like blueprints or shadows.</p></li>
<li><p>Computer Graphics: Rendering 3D scenes onto a 2D screen is fundamentally a projection process.</p></li>
<li><p>Data Science: Projecting high-dimensional data onto a lower-dimensional subspace (like the first two principal components in PCA) makes patterns visible while preserving as much information as possible.</p></li>
<li><p>Signal Processing: Decomposition into projections onto sine and cosine waves forms the basis of Fourier analysis, which powers audio, image, and video compression.</p></li>
</ol>
</section>
<section id="algebraic-properties-4" class="level4">
<h4 class="anchored" data-anchor-id="algebraic-properties-4">Algebraic Properties</h4>
<ul>
<li>Projections are linear: proj(u + w) = proj(u) + proj(w).</li>
<li>The perpendicular part is always orthogonal to the direction of projection.</li>
<li>The decomposition is unique: no other pair of parallel and perpendicular vectors will reconstruct u.</li>
<li>The projection operator onto a unit vector v̂ satisfies: proj(u) = (v̂ v̂ᵀ)u, showing how projection can be expressed in matrix form.</li>
</ul>
</section>
<section id="why-it-matters-7" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-7">Why It Matters</h4>
<p>Projection is not just a geometric trick; it is the core of many advanced topics:</p>
<ul>
<li>Least squares regression is finding the projection of a data vector onto the span of predictor vectors.</li>
<li>Orthogonal decompositions like Gram–Schmidt and QR factorization rely on projections to build orthogonal bases.</li>
<li>Optimization methods often involve projecting guesses back onto feasible sets.</li>
<li>Machine learning uses projections constantly to reduce dimensions, compare vectors, and align features.</li>
</ul>
<p>Without projection, we could not cleanly separate influence along directions or reduce complexity in structured ways.</p>
</section>
<section id="try-it-yourself-7" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-7">Try It Yourself</h4>
<ol type="1">
<li>Project (2, 3) onto (1, 0). What does the perpendicular component look like?</li>
<li>Project (3, 1) onto (2, 2). Verify the perpendicular part is orthogonal.</li>
<li>Decompose (5, 5, 0) into parallel and perpendicular parts relative to (1, 0, 0).</li>
<li>Challenge: Write the projection matrix for projecting onto (1, 2). Apply it to (3, 4). Does it match the formula?</li>
</ol>
<p>Through these exercises, you will see that projection is more than an operation-it is a lens through which we decompose, interpret, and simplify vectors and spaces.</p>
</section>
</section>
<section id="cauchyschwarz-and-triangle-inequalities" class="level3">
<h3 class="anchored" data-anchor-id="cauchyschwarz-and-triangle-inequalities">9. Cauchy–Schwarz and Triangle Inequalities</h3>
<p>Linear algebra is not only about operations with vectors-it also involves understanding the fundamental relationships between them. Two of the most important results in this regard are the Cauchy–Schwarz inequality and the triangle inequality. These are cornerstones of vector spaces because they establish precise boundaries for lengths, angles, and inner products. Without them, the geometry of linear algebra would fall apart.</p>
<section id="the-cauchyschwarz-inequality" class="level4">
<h4 class="anchored" data-anchor-id="the-cauchyschwarz-inequality">The Cauchy–Schwarz Inequality</h4>
<p>For any two vectors <span class="math inline">\(u\)</span> and <span class="math inline">\(v\)</span> in <span class="math inline">\(\mathbb{R}^n\)</span>, the Cauchy–Schwarz inequality states:</p>
<p><span class="math display">\[
|u \cdot v| \leq \|u\| \, \|v\|.
\]</span></p>
<p>This means that the absolute value of the dot product of two vectors is always less than or equal to the product of their lengths.</p>
<p>Equality holds if and only if u and v are linearly dependent (i.e., one is a scalar multiple of the other).</p>
<section id="why-it-is-true" class="level5">
<h5 class="anchored" data-anchor-id="why-it-is-true">Why It Is True</h5>
<p>Recall the geometric formula for the dot product:</p>
<p><span class="math display">\[
u \cdot v = \|u\| \, \|v\| \cos(\theta).
\]</span></p>
<p>Since <span class="math inline">\(-1 \leq \cos(\theta) \leq 1\)</span>, the magnitude of the dot product cannot exceed <span class="math inline">\(\|u\| \, \|v\|\)</span>.<br>
This is exactly the inequality.</p>
</section>
<section id="example" class="level5">
<h5 class="anchored" data-anchor-id="example">Example</h5>
<p>Let <span class="math inline">\(u = (3, 4)\)</span> and <span class="math inline">\(v = (-4, 3)\)</span>.</p>
<ul>
<li>Dot product: <span class="math inline">\((3 \times -4) + (4 \times 3) = -12 + 12 = 0\)</span><br>
</li>
<li>Norms: <span class="math inline">\(\|u\| = 5\)</span>, <span class="math inline">\(\|v\| = 5\)</span><br>
</li>
<li>Product of norms: <span class="math inline">\(25\)</span><br>
</li>
<li><span class="math inline">\(|u \cdot v| = 0 \leq 25\)</span>, which satisfies the inequality</li>
</ul>
<p>Equality does not hold since they are not multiples - they are perpendicular.</p>
</section>
<section id="intuition" class="level5">
<h5 class="anchored" data-anchor-id="intuition">Intuition</h5>
<p>The inequality tells us that two vectors can never “overlap” more strongly than the product of their magnitudes. If they align perfectly, the overlap is maximum (equality). If they’re perpendicular, the overlap is zero.</p>
<p>Think of it as: “the shadow of one vector on another can never be longer than the vector itself.”</p>
</section>
</section>
<section id="the-triangle-inequality" class="level4">
<h4 class="anchored" data-anchor-id="the-triangle-inequality">The Triangle Inequality</h4>
<p>For any vectors <span class="math inline">\(u\)</span> and <span class="math inline">\(v\)</span>, the triangle inequality states:</p>
<p><span class="math display">\[
\|u + v\| \leq \|u\| + \|v\|.
\]</span></p>
<p>This mirrors the geometric fact that in a triangle, any side is at most as long as the sum of the other two sides.</p>
<section id="example-1" class="level5">
<h5 class="anchored" data-anchor-id="example-1">Example</h5>
<p>Let <span class="math inline">\(u = (1, 2)\)</span> and <span class="math inline">\(v = (3, 4)\)</span>.</p>
<ul>
<li><span class="math inline">\(\|u + v\| = \|(4, 6)\| = \sqrt{16 + 36} = \sqrt{52} \approx 7.21\)</span><br>
</li>
<li><span class="math inline">\(\|u\| + \|v\| = \sqrt{5} + 5 \approx 2.24 + 5 = 7.24\)</span></li>
</ul>
<p>Indeed, <span class="math inline">\(7.21 \leq 7.24\)</span>, very close in this case.</p>
</section>
<section id="equality-case" class="level5">
<h5 class="anchored" data-anchor-id="equality-case">Equality Case</h5>
<p>The triangle inequality becomes equality when the vectors point in exactly the same direction (or are scalar multiples with nonnegative coefficients). For example, (1, 1) and (2, 2) produce equality because adding them gives a vector whose length equals the sum of their lengths.</p>
</section>
</section>
<section id="extensions" class="level4">
<h4 class="anchored" data-anchor-id="extensions">Extensions</h4>
<ul>
<li>These inequalities hold in all inner product spaces, not just ℝⁿ. This means they apply to functions, sequences, and more abstract mathematical objects.</li>
<li>In Hilbert spaces (infinite-dimensional generalizations), they remain just as essential.</li>
</ul>
</section>
<section id="why-they-matter" class="level4">
<h4 class="anchored" data-anchor-id="why-they-matter">Why They Matter</h4>
<ol type="1">
<li>They guarantee that the dot product and norm are well-behaved and geometrically meaningful.</li>
<li>They ensure that the norm satisfies the requirements of a distance measure: nonnegativity, symmetry, and triangle inequality.</li>
<li>They underpin the validity of projections, orthogonality, and least squares methods.</li>
<li>They are essential in proving convergence of algorithms, error bounds, and stability in numerical linear algebra.</li>
</ol>
<p>Without these inequalities, we could not trust that the geometry of vector spaces behaves consistently.</p>
</section>
<section id="try-it-yourself-8" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-8">Try It Yourself</h4>
<ol type="1">
<li>Verify Cauchy–Schwarz for (2, –1, 3) and (–1, 4, 0). Compute both sides.</li>
<li>Try the triangle inequality for (–3, 4) and (5, –12). Does equality hold?</li>
<li>Find two vectors where Cauchy–Schwarz is an equality. Explain why.</li>
<li>Challenge: Prove the triangle inequality in <span class="math inline">\(\mathbb{R}^2\)</span> using only the Pythagorean theorem and algebra, without relying on dot products.</li>
</ol>
<p>Working through these problems will show you why these inequalities are not abstract curiosities but the structural glue of linear algebra’s geometry.</p>
</section>
</section>
<section id="orthonormal-sets-in-mathbbr2-and-mathbbr3" class="level3">
<h3 class="anchored" data-anchor-id="orthonormal-sets-in-mathbbr2-and-mathbbr3">10. Orthonormal sets in <span class="math inline">\(\mathbb{R}^2\)</span> and <span class="math inline">\(\mathbb{R}^3\)</span></h3>
<p>Up to now, we’ve discussed vectors, their lengths, angles, and how to project one onto another. A natural culmination of these ideas is the concept of orthonormal sets. These are collections of vectors that are not only orthogonal (mutually perpendicular) but also normalized (each of length 1). Orthonormal sets form the cleanest, most efficient coordinate systems in linear algebra. They are the mathematical equivalent of having rulers at right angles, perfectly calibrated to unit length.</p>
<section id="orthogonal-and-normalized" class="level4">
<h4 class="anchored" data-anchor-id="orthogonal-and-normalized">Orthogonal and Normalized</h4>
<p>Let’s break the term “orthonormal” into two parts:</p>
<ul>
<li><p>Orthogonal: Two vectors <span class="math inline">\(u\)</span> and <span class="math inline">\(v\)</span> are orthogonal if <span class="math inline">\(u \cdot v = 0\)</span>.<br>
In <span class="math inline">\(\mathbb{R}^2\)</span>, this means the vectors meet at a right angle.<br>
In <span class="math inline">\(\mathbb{R}^3\)</span>, it means they form perpendicular directions.</p></li>
<li><p>Normalized: A vector <span class="math inline">\(v\)</span> is normalized if its length is <span class="math inline">\(1\)</span>, i.e., <span class="math inline">\(\|v\| = 1\)</span>.<br>
Such vectors are called unit vectors.</p></li>
</ul>
<p>When we combine both conditions, we get orthonormal vectors: vectors that are both perpendicular to each other and have unit length.</p>
</section>
<section id="orthonormal-sets-in-mathbbr2" class="level4">
<h4 class="anchored" data-anchor-id="orthonormal-sets-in-mathbbr2">Orthonormal Sets in <span class="math inline">\(\mathbb{R}^2\)</span></h4>
<p>In two dimensions, an orthonormal set typically consists of two vectors.<br>
A classic example is:</p>
<p><span class="math inline">\(e_1 = (1, 0), \quad e_2 = (0, 1)\)</span></p>
<ul>
<li>Dot product: <span class="math inline">\(e_1 \cdot e_2 = (1 \times 0 + 0 \times 1) = 0 \;\;\Rightarrow\;\;\)</span> orthogonal<br>
</li>
<li>Lengths: <span class="math inline">\(\|e_1\| = 1\)</span>, <span class="math inline">\(\|e_2\| = 1 \;\;\Rightarrow\;\;\)</span> normalized</li>
</ul>
<p>Thus, <span class="math inline">\(\{e_1, e_2\}\)</span> is an orthonormal set.<br>
In fact, this is the standard basis for <span class="math inline">\(\mathbb{R}^2\)</span>.<br>
Any vector <span class="math inline">\((x, y)\)</span> can be written as <span class="math inline">\(x e_1 + y e_2\)</span>.<br>
This is the simplest coordinate system.</p>
</section>
<section id="orthonormal-sets-in-mathbbr3" class="level4">
<h4 class="anchored" data-anchor-id="orthonormal-sets-in-mathbbr3">Orthonormal Sets in <span class="math inline">\(\mathbb{R}^3\)</span></h4>
<p>In three dimensions, an orthonormal set usually has three vectors.<br>
The standard basis is:</p>
<p><span class="math inline">\(e_1 = (1, 0, 0), \quad e_2 = (0, 1, 0), \quad e_3 = (0, 0, 1)\)</span></p>
<ul>
<li>Each pair has dot product zero, so they are orthogonal<br>
</li>
<li>Each has length <span class="math inline">\(1\)</span>, so they are normalized<br>
</li>
<li>Together, they span all of <span class="math inline">\(\mathbb{R}^3\)</span></li>
</ul>
<p>Geometrically, they correspond to the <span class="math inline">\(x\)</span>-, <span class="math inline">\(y\)</span>-, and <span class="math inline">\(z\)</span>-axes in 3D space.<br>
Any vector <span class="math inline">\((x, y, z)\)</span> can be written as a linear combination <span class="math inline">\(x e_1 + y e_2 + z e_3\)</span>.</p>
</section>
<section id="beyond-the-standard-basis" class="level4">
<h4 class="anchored" data-anchor-id="beyond-the-standard-basis">Beyond the Standard Basis</h4>
<p>The standard basis is not the only orthonormal set. For example:</p>
<p><span class="math inline">\(u = \left(\tfrac{1}{\sqrt{2}}, \tfrac{1}{\sqrt{2}}\right), \quad
v = \left(-\tfrac{1}{\sqrt{2}}, \tfrac{1}{\sqrt{2}}\right)\)</span></p>
<ul>
<li>Dot product: <span class="math inline">\((\tfrac{1}{\sqrt{2}})(-\tfrac{1}{\sqrt{2}}) + (\tfrac{1}{\sqrt{2}})(\tfrac{1}{\sqrt{2}}) = -\tfrac{1}{2} + \tfrac{1}{2} = 0\)</span><br>
</li>
<li>Lengths: <span class="math inline">\(\sqrt{(\tfrac{1}{\sqrt{2}})^2 + (\tfrac{1}{\sqrt{2}})^2} = \sqrt{\tfrac{1}{2} + \tfrac{1}{2}} = 1\)</span></li>
</ul>
<p>So <span class="math inline">\(\{u, v\}\)</span> is also orthonormal in <span class="math inline">\(\mathbb{R}^2\)</span>.<br>
These vectors are rotated <span class="math inline">\(45^\circ\)</span> relative to the standard axes.</p>
<p>Similarly, in <span class="math inline">\(\mathbb{R}^3\)</span>, you can construct rotated orthonormal sets (such as unit vectors along diagonals), as long as the conditions of perpendicularity and unit length hold.</p>
</section>
<section id="properties-of-orthonormal-sets" class="level4">
<h4 class="anchored" data-anchor-id="properties-of-orthonormal-sets">Properties of Orthonormal Sets</h4>
<ol type="1">
<li><p>Simplified coordinates: If <span class="math inline">\(\{v_1, \ldots, v_k\}\)</span> is an orthonormal set, then for any vector <span class="math inline">\(u\)</span> in their span, the coefficients are easy to compute:<br>
<span class="math display">\[
c_i = u \cdot v_i
\]</span><br>
This is much simpler than solving systems of equations.</p></li>
<li><p>Pythagorean theorem generalized: If vectors are orthonormal, the squared length of their sum is the sum of the squares of their coefficients.<br>
For example, if <span class="math inline">\(u = a v_1 + b v_2\)</span>, then<br>
<span class="math display">\[
\|u\|^2 = a^2 + b^2
\]</span></p></li>
<li><p>Projection is easy: Projecting onto an orthonormal set is straightforward — just take dot products.</p></li>
<li><p>Matrices become nice: When vectors form the columns of a matrix, orthonormality makes that matrix an orthogonal matrix, which has special properties: its transpose equals its inverse, and it preserves lengths and angles.</p></li>
</ol>
</section>
<section id="importance-in-mathbbr2-and-mathbbr3" class="level4">
<h4 class="anchored" data-anchor-id="importance-in-mathbbr2-and-mathbbr3">Importance in <span class="math inline">\(\mathbb{R}^2\)</span> and <span class="math inline">\(\mathbb{R}^3\)</span></h4>
<ul>
<li>In geometry, orthonormal bases correspond to coordinate axes.</li>
<li>In physics, they represent independent directions of motion or force.</li>
<li>In computer graphics, orthonormal sets define camera axes and object rotations.</li>
<li>In engineering, they simplify stress, strain, and rotation analysis.</li>
</ul>
<p>Even though <span class="math inline">\(\mathbb{R}^2\)</span> and <span class="math inline">\(\mathbb{R}^3\)</span> are relatively simple, the same ideas extend naturally to higher dimensions, where visualization is impossible but the algebra is identical.</p>
</section>
<section id="why-orthonormal-sets-matter" class="level4">
<h4 class="anchored" data-anchor-id="why-orthonormal-sets-matter">Why Orthonormal Sets Matter</h4>
<p>Orthonormality is the gold standard for building bases in linear algebra:</p>
<ul>
<li>It makes calculations fast and simple.</li>
<li>It ensures numerical stability in computations (important in algorithms and simulations).</li>
<li>It underpins key decompositions like QR factorization, singular value decomposition (SVD), and spectral theorems.</li>
<li>It provides the cleanest way to think about space: orthogonal, independent directions scaled to unit length.</li>
</ul>
<p>Whenever possible, mathematicians and engineers prefer orthonormal bases over arbitrary ones.</p>
</section>
<section id="try-it-yourself-9" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-9">Try It Yourself</h4>
<ol type="1">
<li>Verify that (3/5, 4/5) and (–4/5, 3/5) form an orthonormal set in <span class="math inline">\(\mathbb{R}^2\)</span>.</li>
<li>Construct three orthonormal vectors in <span class="math inline">\(\mathbb{R}^3\)</span> that are not the standard basis. Hint: start with (1/√2, 1/√2, 0) and build perpendiculars.</li>
<li>For u = (2, 1), compute its coordinates relative to the orthonormal set {(1/√2, 1/√2), (–1/√2, 1/√2)}.</li>
<li>Challenge: Prove that if {v₁, …, vₖ} is orthonormal, then the matrix with these as columns is orthogonal, i.e., QᵀQ = I.</li>
</ol>
<p>Through these exercises, you will see how orthonormal sets make every aspect of linear algebra-from projections to decompositions-simpler, cleaner, and more powerful.</p>
</section>
<section id="closing" class="level4">
<h4 class="anchored" data-anchor-id="closing">Closing</h4>
<pre><code>Lengths, angles revealed,
projections trace hidden lines,
clarity takes shape.</code></pre>
</section>
</section>
</section>
<section id="chapter-2.-matrices-and-basic-operations" class="level2">
<h2 class="anchored" data-anchor-id="chapter-2.-matrices-and-basic-operations">Chapter 2. Matrices and basic operations</h2>
<section id="opening-1" class="level4">
<h4 class="anchored" data-anchor-id="opening-1">Opening</h4>
<pre><code>Rows and columns meet,
woven grids of silent rules,
machines of order.</code></pre>
</section>
<section id="matrices-as-tables-and-as-machines" class="level3">
<h3 class="anchored" data-anchor-id="matrices-as-tables-and-as-machines">11. Matrices as Tables and as Machines</h3>
<p>The next stage in our journey is to move from vectors to matrices. A matrix may look like just a rectangular array of numbers, but in linear algebra it plays two distinct and equally important roles:</p>
<ol type="1">
<li>As a table of numbers, storing data, coefficients, or geometric patterns in a compact form.</li>
<li>As a machine that transforms vectors into other vectors, capturing the essence of linear transformations.</li>
</ol>
<p>Both views are valid, and learning to switch between them is crucial to building intuition.</p>
<section id="matrices-as-tables" class="level4">
<h4 class="anchored" data-anchor-id="matrices-as-tables">Matrices as Tables</h4>
<p>At the most basic level, a matrix is a grid of numbers arranged into rows and columns.</p>
<ul>
<li><p>A <span class="math inline">\(2 \times 2\)</span> matrix has 2 rows and 2 columns:</p>
<p><span class="math display">\[
A = \begin{bmatrix}
a_{11} &amp; a_{12} \\
a_{21} &amp; a_{22}
\end{bmatrix}
\]</span></p></li>
<li><p>A <span class="math inline">\(3 \times 2\)</span> matrix has 3 rows and 2 columns:</p>
<p><span class="math display">\[
B = \begin{bmatrix}
b_{11} &amp; b_{12} \\
b_{21} &amp; b_{22} \\
b_{31} &amp; b_{32}
\end{bmatrix}
\]</span></p></li>
</ul>
<p>Each entry <span class="math inline">\(a_{ij}\)</span> or <span class="math inline">\(b_{ij}\)</span> tells us the number in the i-th row and j-th column. The rows of a matrix can represent constraints, equations, or observations; the columns can represent features, variables, or directions.</p>
<p>In this sense, matrices are data containers, organizing information efficiently. That’s why matrices show up in spreadsheets, statistics, computer graphics, and scientific computing.</p>
</section>
<section id="matrices-as-machines" class="level4">
<h4 class="anchored" data-anchor-id="matrices-as-machines">Matrices as Machines</h4>
<p>The deeper view of a matrix is as a function from vectors to vectors. If x is a column vector, then multiplying A·x produces a new vector.</p>
<p>For example:</p>
<p><span class="math display">\[
A = \begin{bmatrix}
2 &amp; 0 \\
1 &amp; 3
\end{bmatrix}, \quad
\mathbf{x} = \begin{bmatrix}
4 \\
5
\end{bmatrix}.
\]</span></p>
<p>Multiplying:</p>
<p><span class="math display">\[
A\mathbf{x} = \begin{bmatrix}
2×4 + 0×5 \\
1×4 + 3×5
\end{bmatrix}
= \begin{bmatrix}
8 \\
19
\end{bmatrix}.
\]</span></p>
<p>Here, the matrix is acting as a machine that takes input (4, 5) and outputs (8, 19). The “machine rules” are encoded in the rows of A.</p>
</section>
<section id="column-view-of-matrix-multiplication" class="level4">
<h4 class="anchored" data-anchor-id="column-view-of-matrix-multiplication">Column View of Matrix Multiplication</h4>
<p>Another way to see it: multiplying A·x is the same as taking a linear combination of A’s columns.</p>
<p>If</p>
<p><span class="math display">\[
A = \begin{bmatrix}
a_1 &amp; a_2
\end{bmatrix}, \quad \mathbf{x} = \begin{bmatrix}
x_1 \\
x_2
\end{bmatrix},
\]</span></p>
<p>then:</p>
<p><span class="math display">\[
A\mathbf{x} = x_1 a_1 + x_2 a_2.
\]</span></p>
<p>So the vector x tells the machine “how much” of each column to mix together. This column view is critical-it connects matrices to span, dimension, and basis ideas we saw earlier.</p>
</section>
<section id="the-duality-of-tables-and-machines" class="level4">
<h4 class="anchored" data-anchor-id="the-duality-of-tables-and-machines">The Duality of Tables and Machines</h4>
<ul>
<li>As a table, a matrix is a static object: numbers written in rows and columns.</li>
<li>As a machine, the same numbers become instructions for transforming vectors.</li>
</ul>
<p>This duality is not just conceptual-it’s the key to understanding why linear algebra is so powerful. A dataset, once stored as a table, can be interpreted as a transformation. Likewise, a transformation, once understood, can be encoded as a table.</p>
</section>
<section id="examples-in-practice" class="level4">
<h4 class="anchored" data-anchor-id="examples-in-practice">Examples in Practice</h4>
<ol type="1">
<li>Physics: A stress–strain matrix is a table of coefficients. But it also acts as a machine that transforms applied forces into deformations.</li>
<li>Computer Graphics: A 2D rotation matrix is a machine that spins vectors, but it can be stored in a simple 2×2 table.</li>
<li>Economics: Input–output models use matrices as tables of production coefficients. Applying them to demand vectors transforms them into resource requirements.</li>
</ol>
</section>
<section id="geometric-intuition" class="level4">
<h4 class="anchored" data-anchor-id="geometric-intuition">Geometric Intuition</h4>
<p>Every 2×2 or 3×3 matrix corresponds to some linear transformation in the plane or space. Examples:</p>
<ul>
<li>Scaling: <span class="math inline">\(\begin{bmatrix} 2 &amp; 0 \\ 0 &amp; 2 \end{bmatrix}\)</span> doubles lengths.</li>
<li>Reflection: <span class="math inline">\(\begin{bmatrix} 1 &amp; 0 \\ 0 &amp; -1 \end{bmatrix}\)</span> flips across the x-axis.</li>
<li>Rotation: <span class="math inline">\(\begin{bmatrix} \cos θ &amp; -\sin θ \\ \sin θ &amp; \cos θ \end{bmatrix}\)</span> rotates vectors by θ.</li>
</ul>
<p>These are not just tables of numbers-they are precise, reusable machines.</p>
</section>
<section id="why-this-matters" class="level4">
<h4 class="anchored" data-anchor-id="why-this-matters">Why This Matters</h4>
<p>This section sets the stage for all matrix theory:</p>
<ul>
<li>Thinking of matrices as tables helps in data interpretation and organization.</li>
<li>Thinking of matrices as machines helps in understanding linear transformations, eigenvalues, and decompositions.</li>
<li>Most importantly, learning to switch between the two perspectives makes linear algebra both concrete and abstract-bridging computation with geometry.</li>
</ul>
</section>
<section id="try-it-yourself-10" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-10">Try It Yourself</h4>
<ol type="1">
<li>Write a 2×3 matrix and identify its rows and columns. What might they represent in a real-world dataset?</li>
<li>Multiply <span class="math inline">\(\begin{bmatrix} 1 &amp; 2 \\ 3 &amp; 4 \end{bmatrix}\)</span> by <span class="math inline">\(\begin{bmatrix} 2 \\ –1 \end{bmatrix}\)</span>. Interpret the result using both the row and column views.</li>
<li>Construct a matrix that scales vectors by 2 along the x-axis and reflects them across the y-axis. Test it on (1, 1).</li>
<li>Challenge: Show how the same 3×3 rotation matrix can be viewed as a data table of cosines/sines and as a machine that turns input vectors.</li>
</ol>
<p>By mastering both perspectives, you’ll see matrices not just as numbers but as dynamic objects that encode and execute transformations.</p>
</section>
</section>
<section id="matrix-shapes-indexing-and-block-views" class="level3">
<h3 class="anchored" data-anchor-id="matrix-shapes-indexing-and-block-views">12. Matrix Shapes, Indexing, and Block Views</h3>
<p>Matrices come in many shapes and sizes, and the way we label their entries matters. This section is about learning how to read and write matrices carefully, how to work with rows and columns, and how to use block structure to simplify problems. These seemingly simple ideas are what allow us to manipulate large systems with precision and efficiency.</p>
<section id="shapes-of-matrices" class="level4">
<h4 class="anchored" data-anchor-id="shapes-of-matrices">Shapes of Matrices</h4>
<p>The shape of a matrix is given by its number of rows and columns:</p>
<ul>
<li>A m×n matrix has m rows and n columns.</li>
<li>Rows run horizontally, columns run vertically.</li>
<li>Square matrices have m = n; rectangular matrices have m ≠ n.</li>
</ul>
<p>Examples:</p>
<ul>
<li><p>A 2×3 matrix:</p>
<p><span class="math display">\[
\begin{bmatrix}
1 &amp; 2 &amp; 3 \\
4 &amp; 5 &amp; 6
\end{bmatrix}
\]</span></p></li>
<li><p>A 3×2 matrix:</p>
<p><span class="math display">\[
\begin{bmatrix}
7 &amp; 8 \\
9 &amp; 10 \\
11 &amp; 12
\end{bmatrix}
\]</span></p></li>
</ul>
<p>Shape matters because it determines whether certain operations (like multiplication) are possible.</p>
</section>
<section id="indexing-the-language-of-entries" class="level4">
<h4 class="anchored" data-anchor-id="indexing-the-language-of-entries">Indexing: The Language of Entries</h4>
<p>Each entry in a matrix has two indices: one for its row, one for its column.</p>
<ul>
<li><span class="math inline">\(a_{ij}\)</span> = entry in row i, column j.</li>
<li>The first index always refers to the row, the second to the column.</li>
</ul>
<p>For example, in</p>
<p><span class="math display">\[
A = \begin{bmatrix}
1 &amp; 4 &amp; 7 \\
2 &amp; 5 &amp; 8 \\
3 &amp; 6 &amp; 9
\end{bmatrix},
\]</span></p>
<p>we have:</p>
<ul>
<li><span class="math inline">\(a_{11} = 1\)</span>, <span class="math inline">\(a_{23} = 8\)</span>, <span class="math inline">\(a_{32} = 6\)</span>.</li>
</ul>
<p>Indexing is the grammar of matrix language. Without it, we can’t specify positions or write formulas clearly.</p>
</section>
<section id="rows-and-columns-as-vectors" class="level4">
<h4 class="anchored" data-anchor-id="rows-and-columns-as-vectors">Rows and Columns as Vectors</h4>
<p>Every row and every column of a matrix is itself a vector.</p>
<ul>
<li>The i-th row is written as <span class="math inline">\(A_{i,*}\)</span>.</li>
<li>The j-th column is written as <span class="math inline">\(A_{*,j}\)</span>.</li>
</ul>
<p>Example: From the matrix above,</p>
<ul>
<li>First row: (1, 4, 7).</li>
<li>Second column: (4, 5, 6).</li>
</ul>
<p>This duality is powerful: rows often represent constraints or equations, while columns represent directions or features. Later, when we interpret matrix–vector products, we’ll see that multiplying A·x means combining columns, while multiplying yᵀ·A means combining rows.</p>
</section>
<section id="submatrices" class="level4">
<h4 class="anchored" data-anchor-id="submatrices">Submatrices</h4>
<p>Sometimes we want just part of a matrix. A submatrix is formed by selecting certain rows and columns.</p>
<p>Example: From</p>
<p><span class="math display">\[
B = \begin{bmatrix}
2 &amp; 4 &amp; 6 \\
1 &amp; 3 &amp; 5 \\
7 &amp; 8 &amp; 9
\end{bmatrix},
\]</span></p>
<p>the submatrix of the first two rows and last two columns is:</p>
<p><span class="math display">\[
\begin{bmatrix}
4 &amp; 6 \\
3 &amp; 5
\end{bmatrix}.
\]</span></p>
<p>Submatrices allow us to zoom in and isolate parts of a problem.</p>
</section>
<section id="block-matrices-dividing-to-conquer" class="level4">
<h4 class="anchored" data-anchor-id="block-matrices-dividing-to-conquer">Block Matrices: Dividing to Conquer</h4>
<p>Large matrices can often be broken into blocks, which are smaller submatrices arranged inside. This is like dividing a spreadsheet into quadrants.</p>
<p>For example:</p>
<p><span class="math display">\[
C = \begin{bmatrix}
A_{11} &amp; A_{12} \\
A_{21} &amp; A_{22}
\end{bmatrix},
\]</span></p>
<p>where each <span class="math inline">\(A_{ij}\)</span> is itself a smaller matrix.</p>
<p>This structure is useful in:</p>
<ul>
<li>Computation: Algorithms often process blocks instead of individual entries.</li>
<li>Theory: Many proofs and factorizations rely on viewing a matrix in blocks (e.g., LU, QR, Schur decomposition).</li>
<li>Applications: Partitioning data tables into logical sections.</li>
</ul>
<p>Example: Splitting a 4×4 matrix into four 2×2 blocks helps us treat it as a “matrix of matrices.”</p>
</section>
<section id="special-shapes" class="level4">
<h4 class="anchored" data-anchor-id="special-shapes">Special Shapes</h4>
<p>Some shapes of matrices are so common they deserve names:</p>
<ul>
<li>Row vector: 1×n matrix.</li>
<li>Column vector: n×1 matrix.</li>
<li>Diagonal matrix: Nonzero entries only on the diagonal.</li>
<li>Identity matrix: Square diagonal matrix with 1’s on the diagonal.</li>
<li>Zero matrix: All entries are 0.</li>
</ul>
<p>Recognizing these shapes saves time and clarifies reasoning.</p>
</section>
<section id="why-it-matters-8" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-8">Why It Matters</h4>
<p>Careful attention to matrix shapes, indexing, and block views ensures:</p>
<ol type="1">
<li>Precision: We can describe positions unambiguously.</li>
<li>Structure awareness: Recognizing patterns (diagonal, triangular, block) leads to more efficient computations.</li>
<li>Scalability: Block partitioning is the foundation of modern numerical linear algebra libraries, where matrices are too large to handle entry by entry.</li>
<li>Geometry: Rows and columns as vectors connect matrix structure to span, basis, and dimension.</li>
</ol>
<p>These basic tools prepare us for multiplication, transformations, and factorization.</p>
</section>
<section id="try-it-yourself-11" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-11">Try It Yourself</h4>
<ol type="1">
<li><p>Write a 3×4 matrix and label the entry in row 2, column 3.</p></li>
<li><p>Extract a 2×2 submatrix from the corners of a 4×4 matrix of your choice.</p></li>
<li><p>Break a 6×6 matrix into four 3×3 blocks. How would you represent it compactly?</p></li>
<li><p>Challenge: Given</p>
<p><span class="math display">\[
D = \begin{bmatrix}
1 &amp; 2 &amp; 3 &amp; 4 \\
5 &amp; 6 &amp; 7 &amp; 8 \\
9 &amp; 10 &amp; 11 &amp; 12
\end{bmatrix},
\]</span></p>
<p>write it as a block matrix with a 2×2 block in the top-left, a 2×2 block in the top-right, and a 1×4 block in the bottom row.</p></li>
</ol>
<p>By practicing with shapes, indexing, and blocks, you’ll develop the ability to navigate matrices not just as raw grids of numbers but as structured objects ready for deeper algebraic and geometric insights.</p>
</section>
</section>
<section id="matrix-addition-and-scalar-multiplication" class="level3">
<h3 class="anchored" data-anchor-id="matrix-addition-and-scalar-multiplication">13. Matrix Addition and Scalar Multiplication</h3>
<p>Before exploring matrix–vector and matrix–matrix multiplication, it is essential to understand the simplest operations we can perform with matrices: addition and scalar multiplication. These operations extend the rules we learned for vectors, but now applied to entire grids of numbers. Although straightforward, they are the foundation for more complex algebraic manipulations and help establish the idea of matrices as elements of a vector space.</p>
<section id="matrix-addition-entry-by-entry" class="level4">
<h4 class="anchored" data-anchor-id="matrix-addition-entry-by-entry">Matrix Addition: Entry by Entry</h4>
<p>If two matrices <span class="math inline">\(A\)</span> and <span class="math inline">\(B\)</span> have the same shape (same number of rows and columns), we can add them by adding corresponding entries.</p>
<p>Formally: If</p>
<p><span class="math display">\[
A = [a_{ij}], \quad B = [b_{ij}],
\]</span></p>
<p>then</p>
<p><span class="math display">\[
A + B = [a_{ij} + b_{ij}].
\]</span></p>
<p>Example:</p>
<p><span class="math display">\[
\begin{bmatrix}
1 &amp; 2 &amp; 3 \\
4 &amp; 5 &amp; 6
\end{bmatrix}
+
\begin{bmatrix}
7 &amp; 8 &amp; 9 \\
10 &amp; 11 &amp; 12
\end{bmatrix}
=
\begin{bmatrix}
8 &amp; 10 &amp; 12 \\
14 &amp; 16 &amp; 18
\end{bmatrix}.
\]</span></p>
<p>Key point: Addition is only defined if the matrices are the same shape. A 2×3 matrix cannot be added to a 3×2 matrix.</p>
</section>
<section id="scalar-multiplication-scaling-every-entry" class="level4">
<h4 class="anchored" data-anchor-id="scalar-multiplication-scaling-every-entry">Scalar Multiplication: Scaling Every Entry</h4>
<p>A scalar multiplies every entry of a matrix.</p>
<p>Formally: For scalar <span class="math inline">\(c\)</span> and matrix <span class="math inline">\(A = [a_{ij}]\)</span>,</p>
<p><span class="math display">\[
cA = [c \cdot a_{ij}].
\]</span></p>
<p>Example:</p>
<p><span class="math display">\[
3 \cdot
\begin{bmatrix}
2 &amp; -1 \\
0 &amp; 4
\end{bmatrix}
=
\begin{bmatrix}
6 &amp; -3 \\
0 &amp; 12
\end{bmatrix}.
\]</span></p>
<p>This mirrors vector scaling: stretching or shrinking the whole matrix by a constant factor.</p>
</section>
<section id="properties-of-addition-and-scalar-multiplication" class="level4">
<h4 class="anchored" data-anchor-id="properties-of-addition-and-scalar-multiplication">Properties of Addition and Scalar Multiplication</h4>
<p>These two operations satisfy familiar algebraic properties that make the set of all m×n matrices into a vector space:</p>
<ol type="1">
<li>Commutativity: <span class="math inline">\(A + B = B + A\)</span>.</li>
<li>Associativity: <span class="math inline">\((A + B) + C = A + (B + C)\)</span>.</li>
<li>Additive identity: <span class="math inline">\(A + 0 = A\)</span>, where 0 is the zero matrix.</li>
<li>Additive inverse: For every <span class="math inline">\(A\)</span>, there exists <span class="math inline">\(-A\)</span> such that <span class="math inline">\(A + (-A) = 0\)</span>.</li>
<li>Distributivity: <span class="math inline">\(c(A + B) = cA + cB\)</span>.</li>
<li>Compatibility: <span class="math inline">\((c + d)A = cA + dA\)</span>.</li>
<li>Scalar associativity: <span class="math inline">\((cd)A = c(dA)\)</span>.</li>
<li>Unit scalar: <span class="math inline">\(1A = A\)</span>.</li>
</ol>
<p>These guarantee that working with matrices feels like working with numbers and vectors, only in a higher-level setting.</p>
</section>
<section id="matrix-arithmetic-as-table-operations" class="level4">
<h4 class="anchored" data-anchor-id="matrix-arithmetic-as-table-operations">Matrix Arithmetic as Table Operations</h4>
<p>From the table view, addition and scalar multiplication are just simple bookkeeping: line up two tables of the same shape and add entry by entry; multiply the whole table by a constant.</p>
<p>Example: Imagine two spreadsheets of monthly expenses. Adding them gives combined totals. Multiplying by 12 converts a monthly table into a yearly estimate.</p>
</section>
<section id="matrix-arithmetic-as-machine-operations" class="level4">
<h4 class="anchored" data-anchor-id="matrix-arithmetic-as-machine-operations">Matrix Arithmetic as Machine Operations</h4>
<p>From the machine view, these operations adjust the behavior of linear transformations:</p>
<ul>
<li>Adding matrices corresponds to adding their effects when applied to vectors.</li>
<li>Scaling a matrix scales the effect of the transformation.</li>
</ul>
<p>Example: Let <span class="math inline">\(A\)</span> rotate vectors slightly, and <span class="math inline">\(B\)</span> stretch vectors. The matrix <span class="math inline">\(A + B\)</span> represents a transformation that applies both influences together. Scaling by 2 doubles the effect of the transformation.</p>
</section>
<section id="special-case-zero-and-identity" class="level4">
<h4 class="anchored" data-anchor-id="special-case-zero-and-identity">Special Case: Zero and Identity</h4>
<ul>
<li>Zero matrix: All entries are 0. Adding it to any matrix changes nothing.</li>
<li>Scalar multiples of the identity: <span class="math inline">\(cI\)</span> scales every vector by c when applied. For example, <span class="math inline">\(2I\)</span> doubles every vector’s length.</li>
</ul>
<p>These act as neutral or scaling elements in matrix arithmetic.</p>
</section>
<section id="geometric-intuition-1" class="level4">
<h4 class="anchored" data-anchor-id="geometric-intuition-1">Geometric Intuition</h4>
<ol type="1">
<li>In <span class="math inline">\(\mathbb{R}^2\)</span> or <span class="math inline">\(\mathbb{R}^3\)</span>, adding transformation matrices is like superimposing geometric effects: e.g., one matrix shears, another rotates, their sum mixes both.</li>
<li>Scaling a transformation makes its action stronger or weaker. Doubling a shear makes it twice as pronounced.</li>
</ol>
<p>This shows that even before multiplication, addition and scaling already have geometric meaning.</p>
</section>
<section id="why-it-matters-9" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-9">Why It Matters</h4>
<p>Though simple, these operations:</p>
<ul>
<li>Define matrices as elements of vector spaces.</li>
<li>Lay the groundwork for linear combinations of matrices, critical in eigenvalue problems, optimization, and control theory.</li>
<li>Enable modular problem-solving: break big transformations into smaller ones and recombine them.</li>
<li>Appear everywhere in practice, from combining datasets to scaling transformations.</li>
</ul>
<p>Without addition and scalar multiplication, we could not treat matrices systematically as algebraic objects.</p>
</section>
<section id="try-it-yourself-12" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-12">Try It Yourself</h4>
<ol type="1">
<li>Add</li>
</ol>
<p><span class="math display">\[
\begin{bmatrix}
2 &amp; 0 \\
1 &amp; 3
\end{bmatrix}
\quad \text{and} \quad
\begin{bmatrix}
-2 &amp; 5 \\
4 &amp; -3
\end{bmatrix}.
\]</span></p>
<ol start="2" type="1">
<li>Multiply</li>
</ol>
<p><span class="math display">\[
\begin{bmatrix}
1 &amp; -1 &amp; 2 \\
0 &amp; 3 &amp; 4
\end{bmatrix}
\]</span></p>
<p>by –2.</p>
<ol start="3" type="1">
<li><p>Show that (A + B) + C = A + (B + C) with explicit 2×2 matrices.</p></li>
<li><p>Challenge: Construct two 3×3 matrices A and B such that A + B = 0. What does that tell you about B?</p></li>
</ol>
<p>By practicing these fundamentals, you will see that even the most basic operations on matrices already build the algebraic backbone for deeper results like matrix multiplication, transformations, and factorization.</p>
</section>
</section>
<section id="matrixvector-product-linear-combinations-of-columns" class="level3">
<h3 class="anchored" data-anchor-id="matrixvector-product-linear-combinations-of-columns">14. Matrix–Vector Product (Linear Combinations of Columns)</h3>
<p>We now arrive at one of the most important operations in all of linear algebra: the matrix–vector product. This operation takes a matrix <span class="math inline">\(A\)</span> and a vector x, and produces a new vector. While the computation is straightforward, its interpretations are deep: it can be seen as combining rows, as combining columns, or as applying a linear transformation. This is the operation that connects matrices to the geometry of vector spaces.</p>
<section id="the-algebraic-rule" class="level4">
<h4 class="anchored" data-anchor-id="the-algebraic-rule">The Algebraic Rule</h4>
<p>Suppose <span class="math inline">\(A\)</span> is an <span class="math inline">\(m \times n\)</span> matrix, and x is a vector in <span class="math inline">\(\mathbb{R}^n\)</span>. The product <span class="math inline">\(A\mathbf{x}\)</span> is a vector in <span class="math inline">\(\mathbb{R}^m\)</span>, defined as:</p>
<p><span class="math display">\[
A =
\begin{bmatrix}
a_{11} &amp; a_{12} &amp; \cdots &amp; a_{1n} \\
a_{21} &amp; a_{22} &amp; \cdots &amp; a_{2n} \\
\vdots &amp; \vdots &amp; \ddots &amp; \vdots \\
a_{m1} &amp; a_{m2} &amp; \cdots &amp; a_{mn}
\end{bmatrix},
\quad
\mathbf{x} =
\begin{bmatrix}
x_1 \\ x_2 \\ \vdots \\ x_n
\end{bmatrix}.
\]</span></p>
<p>Then:</p>
<p><span class="math display">\[
A\mathbf{x} =
\begin{bmatrix}
a_{11}x_1 + a_{12}x_2 + \cdots + a_{1n}x_n \\
a_{21}x_1 + a_{22}x_2 + \cdots + a_{2n}x_n \\
\vdots \\
a_{m1}x_1 + a_{m2}x_2 + \cdots + a_{mn}x_n
\end{bmatrix}.
\]</span></p>
<p>Each entry of the output is a dot product between one row of <span class="math inline">\(A\)</span> and the vector x.</p>
</section>
<section id="row-view-dot-products" class="level4">
<h4 class="anchored" data-anchor-id="row-view-dot-products">Row View: Dot Products</h4>
<p>From the row perspective, <span class="math inline">\(A\mathbf{x}\)</span> is computed row by row:</p>
<ul>
<li>Take each row of <span class="math inline">\(A\)</span>.</li>
<li>Dot it with x.</li>
<li>That result becomes one entry of the output.</li>
</ul>
<p>Example:</p>
<p><span class="math display">\[
A =
\begin{bmatrix}
2 &amp; 1 \\
3 &amp; 4 \\
-1 &amp; 2
\end{bmatrix}, \quad
\mathbf{x} =
\begin{bmatrix}
5 \\
-1
\end{bmatrix}.
\]</span></p>
<ul>
<li>First row dot x: <span class="math inline">\(2(5) + 1(-1) = 9\)</span>.</li>
<li>Second row dot x: <span class="math inline">\(3(5) + 4(-1) = 11\)</span>.</li>
<li>Third row dot x: <span class="math inline">\((-1)(5) + 2(-1) = -7\)</span>.</li>
</ul>
<p>So:</p>
<p><span class="math display">\[
A\mathbf{x} =
\begin{bmatrix}
9 \\ 11 \\ -7
\end{bmatrix}.
\]</span></p>
</section>
<section id="column-view-linear-combinations" class="level4">
<h4 class="anchored" data-anchor-id="column-view-linear-combinations">Column View: Linear Combinations</h4>
<p>From the column perspective, <span class="math inline">\(A\mathbf{x}\)</span> is a linear combination of the columns of A.</p>
<p>If</p>
<p><span class="math display">\[
A =
\begin{bmatrix}
| &amp; | &amp;  &amp; | \\
a_1 &amp; a_2 &amp; \cdots &amp; a_n \\
| &amp; | &amp;  &amp; |
\end{bmatrix},
\quad
\mathbf{x} =
\begin{bmatrix}
x_1 \\ x_2 \\ \vdots \\ x_n
\end{bmatrix},
\]</span></p>
<p>then:</p>
<p><span class="math display">\[
A\mathbf{x} = x_1 a_1 + x_2 a_2 + \cdots + x_n a_n.
\]</span></p>
<p>That is: multiply each column of <span class="math inline">\(A\)</span> by the corresponding entry in x, then add them up.</p>
<p>This interpretation connects directly to the idea of span: the set of all vectors <span class="math inline">\(A\mathbf{x}\)</span> as x varies is exactly the span of the columns of <span class="math inline">\(A\)</span>.</p>
</section>
<section id="the-machine-view-linear-transformations" class="level4">
<h4 class="anchored" data-anchor-id="the-machine-view-linear-transformations">The Machine View: Linear Transformations</h4>
<p>The machine view ties everything together: multiplying a vector by a matrix means applying the linear transformation represented by the matrix.</p>
<ul>
<li>If <span class="math inline">\(A\)</span> is a 2×2 rotation matrix, then <span class="math inline">\(A\mathbf{x}\)</span> rotates the vector x.</li>
<li>If <span class="math inline">\(A\)</span> is a scaling matrix, then <span class="math inline">\(A\mathbf{x}\)</span> stretches or shrinks x.</li>
<li>If <span class="math inline">\(A\)</span> is a projection matrix, then <span class="math inline">\(A\mathbf{x}\)</span> projects x onto a line or plane.</li>
</ul>
<p>Thus, the algebraic definition encodes geometric and functional meaning.</p>
</section>
<section id="examples-of-geometric-action" class="level4">
<h4 class="anchored" data-anchor-id="examples-of-geometric-action">Examples of Geometric Action</h4>
<ol type="1">
<li>Scaling:</li>
</ol>
<p><span class="math display">\[
A = \begin{bmatrix} 2 &amp; 0 \\ 0 &amp; 2 \end{bmatrix}.
\]</span></p>
<p>Then <span class="math inline">\(A\mathbf{x}\)</span> doubles the length of any vector x.</p>
<ol start="2" type="1">
<li>Reflection:</li>
</ol>
<p><span class="math display">\[
A = \begin{bmatrix} 1 &amp; 0 \\ 0 &amp; -1 \end{bmatrix}.
\]</span></p>
<p>This flips vectors across the x-axis.</p>
<ol start="3" type="1">
<li>Rotation by θ:</li>
</ol>
<p><span class="math display">\[
A = \begin{bmatrix} \cosθ &amp; -\sinθ \\ \sinθ &amp; \cosθ \end{bmatrix}.
\]</span></p>
<p>This rotates vectors counterclockwise by θ in the plane.</p>
</section>
<section id="why-it-matters-10" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-10">Why It Matters</h4>
<p>The matrix–vector product is the building block of everything in linear algebra:</p>
<ol type="1">
<li>It defines the action of a matrix as a linear map.</li>
<li>It connects directly to span and dimension (columns generate all possible outputs).</li>
<li>It underpins solving linear systems, eigenvalue problems, and decompositions.</li>
<li>It is the engine of computation in applied mathematics, from computer graphics to machine learning (e.g., neural networks compute billions of matrix–vector products).</li>
</ol>
</section>
<section id="try-it-yourself-13" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-13">Try It Yourself</h4>
<ol type="1">
<li>Compute</li>
</ol>
<p><span class="math display">\[
\begin{bmatrix}
1 &amp; 2 &amp; 3 \\
4 &amp; 5 &amp; 6
\end{bmatrix}
\begin{bmatrix}
2 \\
0 \\
1
\end{bmatrix}.
\]</span></p>
<ol start="2" type="1">
<li><p>Express the result of the above product as a linear combination of the columns of the matrix.</p></li>
<li><p>Construct a 2×2 matrix that reflects vectors across the line <span class="math inline">\(y = x\)</span>. Test it on (1, 0) and (0, 1).</p></li>
<li><p>Challenge: For a 3×3 matrix, show that the set of all possible <span class="math inline">\(A\mathbf{x}\)</span> (as x varies) is exactly the column space of <span class="math inline">\(A\)</span>.</p></li>
</ol>
<p>By mastering both the computational rules and the interpretations of the matrix–vector product, you will gain the most important insight in linear algebra: matrices are not just tables-they are engines that transform space.</p>
</section>
</section>
<section id="matrixmatrix-product-composition-of-linear-steps" class="level3">
<h3 class="anchored" data-anchor-id="matrixmatrix-product-composition-of-linear-steps">15. Matrix–Matrix Product (Composition of Linear Steps)</h3>
<p>Having understood how a matrix acts on a vector, the next natural step is to understand how one matrix can act on another. This leads us to the matrix–matrix product, a rule for combining two matrices into a single new matrix. Though the arithmetic looks complicated at first, the underlying idea is elegant: multiplying two matrices represents composing two linear transformations.</p>
<section id="the-algebraic-rule-1" class="level4">
<h4 class="anchored" data-anchor-id="the-algebraic-rule-1">The Algebraic Rule</h4>
<p>Suppose <span class="math inline">\(A\)</span> is an <span class="math inline">\(m \times n\)</span> matrix and <span class="math inline">\(B\)</span> is an <span class="math inline">\(n \times p\)</span> matrix. Their product <span class="math inline">\(C = AB\)</span> is an <span class="math inline">\(m \times p\)</span> matrix defined by:</p>
<p><span class="math display">\[
c_{ij} = \sum_{k=1}^n a_{ik} b_{kj}.
\]</span></p>
<p>That is: each entry of <span class="math inline">\(C\)</span> is the dot product of the i-th row of <span class="math inline">\(A\)</span> with the j-th column of <span class="math inline">\(B\)</span>.</p>
</section>
<section id="example-a-23-times-a-32" class="level4">
<h4 class="anchored" data-anchor-id="example-a-23-times-a-32">Example: A 2×3 times a 3×2</h4>
<p><span class="math display">\[
A =
\begin{bmatrix}
1 &amp; 2 &amp; 3 \\
4 &amp; 5 &amp; 6
\end{bmatrix}, \quad
B =
\begin{bmatrix}
7 &amp; 8 \\
9 &amp; 10 \\
11 &amp; 12
\end{bmatrix}.
\]</span></p>
<p>Product: <span class="math inline">\(C = AB\)</span> will be 2×2.</p>
<ul>
<li><span class="math inline">\(c_{11} = 1\cdot 7 + 2\cdot 9 + 3\cdot 11 = 58\)</span>.</li>
<li><span class="math inline">\(c_{12} = 1\cdot 8 + 2\cdot 10 + 3\cdot 12 = 64\)</span>.</li>
<li><span class="math inline">\(c_{21} = 4\cdot 7 + 5\cdot 9 + 6\cdot 11 = 139\)</span>.</li>
<li><span class="math inline">\(c_{22} = 4\cdot 8 + 5\cdot 10 + 6\cdot 12 = 154\)</span>.</li>
</ul>
<p>So:</p>
<p><span class="math display">\[
C =
\begin{bmatrix}
58 &amp; 64 \\
139 &amp; 154
\end{bmatrix}.
\]</span></p>
</section>
<section id="column-view-linear-combinations-of-columns" class="level4">
<h4 class="anchored" data-anchor-id="column-view-linear-combinations-of-columns">Column View: Linear Combinations of Columns</h4>
<p>From the column perspective, <span class="math inline">\(AB\)</span> is computed by applying <span class="math inline">\(A\)</span> to each column of <span class="math inline">\(B\)</span>.</p>
<p>If <span class="math inline">\(B = [b_1 \; b_2 \; \cdots \; b_p]\)</span>, then:</p>
<p><span class="math display">\[
AB = [A b_1 \; A b_2 \; \cdots \; A b_p].
\]</span></p>
<p>That is: multiply <span class="math inline">\(A\)</span> by each column of <span class="math inline">\(B\)</span>. This is often the simplest way to think of the product.</p>
</section>
<section id="row-view-linear-combinations-of-rows" class="level4">
<h4 class="anchored" data-anchor-id="row-view-linear-combinations-of-rows">Row View: Linear Combinations of Rows</h4>
<p>From the row perspective, each row of <span class="math inline">\(AB\)</span> is formed by combining rows of <span class="math inline">\(B\)</span> using coefficients from a row of <span class="math inline">\(A\)</span>. This dual view is less common but equally useful, especially in proofs and algorithms.</p>
</section>
<section id="the-machine-view-composition-of-transformations" class="level4">
<h4 class="anchored" data-anchor-id="the-machine-view-composition-of-transformations">The Machine View: Composition of Transformations</h4>
<p>The most important interpretation is the machine view: multiplying matrices corresponds to composing transformations.</p>
<ul>
<li>If <span class="math inline">\(A\)</span> maps <span class="math inline">\(\mathbb{R}^n \to \mathbb{R}^m\)</span> and <span class="math inline">\(B\)</span> maps <span class="math inline">\(\mathbb{R}^p \to \mathbb{R}^n\)</span>, then <span class="math inline">\(AB\)</span> maps <span class="math inline">\(\mathbb{R}^p \to \mathbb{R}^m\)</span>.</li>
<li>In words: do <span class="math inline">\(B\)</span> first, then <span class="math inline">\(A\)</span>.</li>
</ul>
<p>Example:</p>
<ul>
<li>Let <span class="math inline">\(B\)</span> rotate vectors by 90°.</li>
<li>Let <span class="math inline">\(A\)</span> scale vectors by 2.</li>
<li>Then <span class="math inline">\(AB\)</span> rotates and then scales-both steps combined into a single transformation.</li>
</ul>
</section>
<section id="geometric-examples" class="level4">
<h4 class="anchored" data-anchor-id="geometric-examples">Geometric Examples</h4>
<ol type="1">
<li>Scaling then rotation:</li>
</ol>
<p><span class="math display">\[
A = \begin{bmatrix} 2 &amp; 0 \\ 0 &amp; 2 \end{bmatrix}, \quad
B = \begin{bmatrix} 0 &amp; -1 \\ 1 &amp; 0 \end{bmatrix}.
\]</span></p>
<p>Then <span class="math inline">\(AB\)</span> scales vectors by 2 after rotating them 90°.</p>
<ol start="2" type="1">
<li>Projection then reflection: If <span class="math inline">\(B\)</span> projects onto the x-axis and <span class="math inline">\(A\)</span> reflects across the y-axis, then <span class="math inline">\(AB\)</span> represents “project then reflect.”</li>
</ol>
</section>
<section id="properties-of-matrix-multiplication" class="level4">
<h4 class="anchored" data-anchor-id="properties-of-matrix-multiplication">Properties of Matrix Multiplication</h4>
<ol type="1">
<li>Associative: <span class="math inline">\((AB)C = A(BC)\)</span>.</li>
<li>Distributive: <span class="math inline">\(A(B + C) = AB + AC\)</span>.</li>
<li>Not commutative: In general, <span class="math inline">\(AB \neq BA\)</span>. Order matters!</li>
<li>Identity: <span class="math inline">\(AI = IA = A\)</span>.</li>
</ol>
<p>These properties highlight that while multiplication is structured, it is not symmetric. The order encodes the order of operations in transformations.</p>
</section>
<section id="why-it-matters-11" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-11">Why It Matters</h4>
<p>Matrix multiplication is the core of linear algebra because:</p>
<ol type="1">
<li>It encodes function composition in algebraic form.</li>
<li>It provides a way to capture multiple transformations in a single matrix.</li>
<li>It underpins algorithms in computer graphics, robotics, statistics, and machine learning.</li>
<li>It reveals deeper structure, like commutativity failing, which reflects real-world order of operations.</li>
</ol>
<p>Almost every application of linear algebra-solving equations, computing eigenvalues, training neural networks-relies on efficient matrix multiplication.</p>
</section>
<section id="try-it-yourself-14" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-14">Try It Yourself</h4>
<ol type="1">
<li>Compute</li>
</ol>
<p><span class="math display">\[
\begin{bmatrix}
1 &amp; 0 \\
2 &amp; 3
\end{bmatrix}
\begin{bmatrix}
4 &amp; 5 \\
6 &amp; 7
\end{bmatrix}.
\]</span></p>
<ol start="2" type="1">
<li>Show that <span class="math inline">\(AB \neq BA\)</span> for the matrices</li>
</ol>
<p><span class="math display">\[
A = \begin{bmatrix} 0 &amp; 1 \\ 0 &amp; 0 \end{bmatrix},
\quad
B = \begin{bmatrix} 0 &amp; 0 \\ 1 &amp; 0 \end{bmatrix}.
\]</span></p>
<ol start="3" type="1">
<li><p>Construct two 2×2 matrices where <span class="math inline">\(AB = BA\)</span>. Why does commutativity happen here?</p></li>
<li><p>Challenge: If <span class="math inline">\(A\)</span> is a projection and <span class="math inline">\(B\)</span> is a rotation, compute <span class="math inline">\(AB\)</span> and <span class="math inline">\(BA\)</span>. Do they represent the same geometric operation?</p></li>
</ol>
<p>Through these perspectives, the matrix–matrix product shifts from being a mechanical formula to being a language for combining linear steps-each product telling the story of “do this, then that.”</p>
</section>
</section>
<section id="identity-inverse-and-transpose" class="level3">
<h3 class="anchored" data-anchor-id="identity-inverse-and-transpose">16. Identity, Inverse, and Transpose</h3>
<p>With addition, scalar multiplication, and matrix multiplication in place, we now introduce three special operations and objects that form the backbone of matrix algebra: the identity matrix, the inverse of a matrix, and the transpose of a matrix. Each captures a fundamental principle-neutrality, reversibility, and symmetry-and together they provide the algebraic structure that makes linear algebra so powerful.</p>
<section id="the-identity-matrix" class="level4">
<h4 class="anchored" data-anchor-id="the-identity-matrix">The Identity Matrix</h4>
<p>The identity matrix is the matrix equivalent of the number 1 in multiplication.</p>
<ul>
<li>Definition: The identity matrix <span class="math inline">\(I_n\)</span> is the <span class="math inline">\(n \times n\)</span> matrix with 1’s on the diagonal and 0’s everywhere else.</li>
</ul>
<p>Example (3×3):</p>
<p><span class="math display">\[
I_3 = \begin{bmatrix}
1 &amp; 0 &amp; 0 \\
0 &amp; 1 &amp; 0 \\
0 &amp; 0 &amp; 1
\end{bmatrix}.
\]</span></p>
<ul>
<li><p>Property: For any <span class="math inline">\(n \times n\)</span> matrix <span class="math inline">\(A\)</span>,</p>
<p><span class="math display">\[
AI_n = I_nA = A.
\]</span></p></li>
<li><p>Machine view: <span class="math inline">\(I\)</span> does nothing-it maps every vector to itself.</p></li>
</ul>
</section>
<section id="the-inverse-of-a-matrix" class="level4">
<h4 class="anchored" data-anchor-id="the-inverse-of-a-matrix">The Inverse of a Matrix</h4>
<p>The inverse is the matrix equivalent of the reciprocal of a number.</p>
<ul>
<li><p>Definition: For a square matrix <span class="math inline">\(A\)</span>, its inverse <span class="math inline">\(A^{-1}\)</span> is the matrix such that</p>
<p><span class="math display">\[
AA^{-1} = A^{-1}A = I.
\]</span></p></li>
<li><p>Not all matrices have inverses. A matrix is invertible if and only if it is square and its determinant is nonzero.</p></li>
</ul>
<p>Example:</p>
<p><span class="math display">\[
A = \begin{bmatrix}
2 &amp; 1 \\
1 &amp; 1
\end{bmatrix},
\quad
A^{-1} = \begin{bmatrix}
1 &amp; -1 \\
-1 &amp; 2
\end{bmatrix}.
\]</span></p>
<p>Check:</p>
<p><span class="math display">\[
AA^{-1} = \begin{bmatrix}
2 &amp; 1 \\
1 &amp; 1
\end{bmatrix}
\begin{bmatrix}
1 &amp; -1 \\
-1 &amp; 2
\end{bmatrix}
=
\begin{bmatrix}
1 &amp; 0 \\
0 &amp; 1
\end{bmatrix} = I.
\]</span></p>
<ul>
<li>Machine view: Applying <span class="math inline">\(A\)</span> transforms a vector. Applying <span class="math inline">\(A^{-1}\)</span> undoes that transformation, restoring the original input.</li>
</ul>
</section>
<section id="non-invertible-matrices" class="level4">
<h4 class="anchored" data-anchor-id="non-invertible-matrices">Non-Invertible Matrices</h4>
<p>Some matrices cannot be inverted. These are called singular.</p>
<ul>
<li><p>Example:</p>
<p><span class="math display">\[
B = \begin{bmatrix}
2 &amp; 4 \\
1 &amp; 2
\end{bmatrix}.
\]</span></p>
<p>Here, the second column is a multiple of the first. The transformation squashes vectors into a line, losing information-so it cannot be reversed.</p></li>
</ul>
<p>This ties invertibility to geometry: a transformation that collapses dimensions cannot be undone.</p>
</section>
<section id="the-transpose-of-a-matrix" class="level4">
<h4 class="anchored" data-anchor-id="the-transpose-of-a-matrix">The Transpose of a Matrix</h4>
<p>The transpose reflects a matrix across its diagonal.</p>
<ul>
<li><p>Definition: For <span class="math inline">\(A = [a_{ij}]\)</span>,</p>
<p><span class="math display">\[
A^T = [a_{ji}].
\]</span></p></li>
<li><p>In words: rows become columns, columns become rows.</p></li>
</ul>
<p>Example:</p>
<p><span class="math display">\[
A = \begin{bmatrix}
1 &amp; 2 &amp; 3 \\
4 &amp; 5 &amp; 6
\end{bmatrix},
\quad
A^T = \begin{bmatrix}
1 &amp; 4 \\
2 &amp; 5 \\
3 &amp; 6
\end{bmatrix}.
\]</span></p>
<ul>
<li><p>Properties:</p>
<ul>
<li><span class="math inline">\((A^T)^T = A\)</span>.</li>
<li><span class="math inline">\((A + B)^T = A^T + B^T\)</span>.</li>
<li><span class="math inline">\((cA)^T = cA^T\)</span>.</li>
<li><span class="math inline">\((AB)^T = B^T A^T\)</span> (note the reversed order!).</li>
</ul></li>
</ul>
</section>
<section id="symmetric-and-orthogonal-matrices" class="level4">
<h4 class="anchored" data-anchor-id="symmetric-and-orthogonal-matrices">Symmetric and Orthogonal Matrices</h4>
<p>Two important classes emerge from the transpose:</p>
<ul>
<li><p>Symmetric matrices: <span class="math inline">\(A = A^T\)</span>. Example:</p>
<p><span class="math display">\[
\begin{bmatrix}
2 &amp; 3 \\
3 &amp; 5
\end{bmatrix}.
\]</span></p>
<p>These have beautiful properties: real eigenvalues and orthogonal eigenvectors.</p></li>
<li><p>Orthogonal matrices: <span class="math inline">\(Q^TQ = I\)</span>. Their columns form an orthonormal set, and they represent pure rotations/reflections.</p></li>
</ul>
</section>
<section id="why-it-matters-12" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-12">Why It Matters</h4>
<ol type="1">
<li>The identity guarantees a neutral element for multiplication.</li>
<li>The inverse provides a way to solve equations <span class="math inline">\(A\mathbf{x} = \mathbf{b}\)</span> via <span class="math inline">\(\mathbf{x} = A^{-1}\mathbf{b}\)</span>.</li>
<li>The transpose ties matrices to geometry, inner products, and symmetry.</li>
<li>Together, they form the algebraic foundation for deeper topics: determinants, eigenvalues, factorizations, and numerical methods.</li>
</ol>
<p>Without these tools, matrix algebra would lack structure and reversibility.</p>
</section>
<section id="try-it-yourself-15" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-15">Try It Yourself</h4>
<ol type="1">
<li>Compute the transpose of</li>
</ol>
<p><span class="math display">\[
\begin{bmatrix}
1 &amp; 0 &amp; 2 \\
-3 &amp; 4 &amp; 5
\end{bmatrix}.
\]</span></p>
<ol start="2" type="1">
<li>Verify that <span class="math inline">\((AB)^T = B^TA^T\)</span> for</li>
</ol>
<p><span class="math display">\[
A = \begin{bmatrix} 1 &amp; 2 \\ 0 &amp; 3 \end{bmatrix}, \quad
B = \begin{bmatrix} 4 &amp; 0 \\ 5 &amp; 6 \end{bmatrix}.
\]</span></p>
<ol start="3" type="1">
<li>Find the inverse of</li>
</ol>
<p><span class="math display">\[
\begin{bmatrix}
3 &amp; 2 \\
1 &amp; 1
\end{bmatrix}.
\]</span></p>
<ol start="4" type="1">
<li>Challenge: Show that if <span class="math inline">\(Q\)</span> is orthogonal, then <span class="math inline">\(Q^{-1} = Q^T\)</span>. Interpret this geometrically as saying “rotations can be undone by transposing.”</li>
</ol>
<p>Through these exercises, you’ll see how identity, inverse, and transpose anchor the structure of linear algebra, providing neutrality, reversibility, and symmetry in every calculation.</p>
</section>
</section>
<section id="symmetric-diagonal-triangular-and-permutation-matrices" class="level3">
<h3 class="anchored" data-anchor-id="symmetric-diagonal-triangular-and-permutation-matrices">17. Symmetric, Diagonal, Triangular, and Permutation Matrices</h3>
<p>Not all matrices are created equal-some have special shapes or patterns that give them unique properties. These structured matrices are the workhorses of linear algebra: they simplify computation, reveal geometry, and form the building blocks for algorithms. In this section, we study four especially important classes: symmetric, diagonal, triangular, and permutation matrices.</p>
<section id="symmetric-matrices" class="level4">
<h4 class="anchored" data-anchor-id="symmetric-matrices">Symmetric Matrices</h4>
<p>A matrix is symmetric if it equals its transpose:</p>
<p><span class="math display">\[
A = A^T.
\]</span></p>
<p>Example:</p>
<p><span class="math display">\[
\begin{bmatrix}
2 &amp; 3 &amp; 4 \\
3 &amp; 5 &amp; 6 \\
4 &amp; 6 &amp; 9
\end{bmatrix}.
\]</span></p>
<ul>
<li>Geometric meaning: Symmetric matrices represent linear transformations that have no “handedness.” They often arise in physics (energy, covariance, stiffness).</li>
<li>Algebraic fact: Symmetric matrices have real eigenvalues and an orthonormal basis of eigenvectors. This property underpins the spectral theorem, one of the pillars of linear algebra.</li>
</ul>
</section>
<section id="diagonal-matrices" class="level4">
<h4 class="anchored" data-anchor-id="diagonal-matrices">Diagonal Matrices</h4>
<p>A matrix is diagonal if all non-diagonal entries are zero.</p>
<p><span class="math display">\[
D = \begin{bmatrix}
d_1 &amp; 0 &amp; 0 \\
0 &amp; d_2 &amp; 0 \\
0 &amp; 0 &amp; d_3
\end{bmatrix}.
\]</span></p>
<ul>
<li><p>Multiplying by <span class="math inline">\(D\)</span> scales each coordinate separately.</p></li>
<li><p>Computations with diagonals are lightning fast:</p>
<ul>
<li>Adding: add diagonal entries.</li>
<li>Multiplying: multiply diagonal entries.</li>
<li>Inverting: invert each diagonal entry (if nonzero).</li>
</ul></li>
</ul>
<p>Example:</p>
<p><span class="math display">\[
\begin{bmatrix}
2 &amp; 0 \\
0 &amp; 3
\end{bmatrix}
\begin{bmatrix}
x \\
y
\end{bmatrix}
=
\begin{bmatrix}
2x \\
3y
\end{bmatrix}.
\]</span></p>
<p>This is why diagonalization is so valuable: turning a general matrix into a diagonal one simplifies everything.</p>
</section>
<section id="triangular-matrices" class="level4">
<h4 class="anchored" data-anchor-id="triangular-matrices">Triangular Matrices</h4>
<p>A matrix is upper triangular if all entries below the main diagonal are zero, and lower triangular if all entries above the diagonal are zero.</p>
<ul>
<li><p>Upper triangular example:</p>
<p><span class="math display">\[
\begin{bmatrix}
1 &amp; 2 &amp; 3 \\
0 &amp; 4 &amp; 5 \\
0 &amp; 0 &amp; 6
\end{bmatrix}.
\]</span></p></li>
<li><p>Lower triangular example:</p>
<p><span class="math display">\[
\begin{bmatrix}
7 &amp; 0 &amp; 0 \\
8 &amp; 9 &amp; 0 \\
10 &amp; 11 &amp; 12
\end{bmatrix}.
\]</span></p></li>
</ul>
<p>Why they matter:</p>
<ul>
<li>Determinant = product of diagonal entries.</li>
<li>Easy to solve systems by substitution (forward or backward).</li>
<li>Every square matrix can be factored into triangular matrices (LU decomposition).</li>
</ul>
</section>
<section id="permutation-matrices" class="level4">
<h4 class="anchored" data-anchor-id="permutation-matrices">Permutation Matrices</h4>
<p>A permutation matrix is obtained by permuting the rows (or columns) of an identity matrix.</p>
<p>Example:</p>
<p><span class="math display">\[
P = \begin{bmatrix}
0 &amp; 1 &amp; 0 \\
1 &amp; 0 &amp; 0 \\
0 &amp; 0 &amp; 1
\end{bmatrix}.
\]</span></p>
<p>Multiplying by <span class="math inline">\(P\)</span>:</p>
<ul>
<li>On the left, permutes the rows of a matrix.</li>
<li>On the right, permutes the columns of a matrix.</li>
</ul>
<p>Permutation matrices are used in pivoting strategies in elimination, ensuring numerical stability in solving systems. They are also orthogonal: <span class="math inline">\(P^{-1} = P^T\)</span>.</p>
</section>
<section id="connections-between-them" class="level4">
<h4 class="anchored" data-anchor-id="connections-between-them">Connections Between Them</h4>
<ul>
<li>A diagonal matrix is a special case of triangular (both upper and lower).</li>
<li>Symmetric matrices often become diagonal under orthogonal transformations.</li>
<li>Permutation matrices help reorder triangular or diagonal matrices without breaking their structure.</li>
</ul>
<p>Together, these classes show that structure leads to simplicity-many computational algorithms exploit these patterns for speed and stability.</p>
</section>
<section id="why-it-matters-13" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-13">Why It Matters</h4>
<ol type="1">
<li>Symmetric matrices guarantee stable and interpretable eigen-decompositions.</li>
<li>Diagonal matrices make computation effortless.</li>
<li>Triangular matrices are the backbone of elimination and factorization methods.</li>
<li>Permutation matrices preserve structure while reordering, critical for algorithms.</li>
</ol>
<p>Almost every advanced method in numerical linear algebra relies on reducing general matrices into one of these structured forms.</p>
</section>
<section id="try-it-yourself-16" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-16">Try It Yourself</h4>
<ol type="1">
<li>Verify that</li>
</ol>
<p><span class="math display">\[
\begin{bmatrix}
1 &amp; 2 \\
2 &amp; 5
\end{bmatrix}
\]</span></p>
<p>is symmetric. Find its transpose.</p>
<ol start="2" type="1">
<li>Compute the determinant of</li>
</ol>
<p><span class="math display">\[
\begin{bmatrix}
3 &amp; 0 &amp; 0 \\
0 &amp; 4 &amp; 0 \\
0 &amp; 0 &amp; 5
\end{bmatrix}.
\]</span></p>
<ol start="3" type="1">
<li>Solve</li>
</ol>
<p><span class="math display">\[
\begin{bmatrix}
2 &amp; 3 &amp; 1 \\
0 &amp; 5 &amp; 2 \\
0 &amp; 0 &amp; 4
\end{bmatrix}
\mathbf{x} =
\begin{bmatrix}
1 \\
2 \\
3
\end{bmatrix}
\]</span></p>
<p>using back substitution.</p>
<ol start="4" type="1">
<li>Construct a 4×4 permutation matrix that swaps the first and last rows. Apply it to a 4×1 vector of your choice.</li>
</ol>
<p>By exploring these four structured families, you’ll start to see that not all matrices are messy-many have order hidden in their arrangement, and exploiting that order is the key to both theoretical understanding and efficient computation.</p>
</section>
</section>
<section id="trace-and-basic-matrix-properties" class="level3">
<h3 class="anchored" data-anchor-id="trace-and-basic-matrix-properties">18. Trace and Basic Matrix Properties</h3>
<p>So far we have studied shapes, multiplication rules, and special classes of matrices. In this section we introduce a simple but surprisingly powerful quantity: the trace of a matrix. Along with it, we review a set of basic matrix properties that provide shortcuts, invariants, and insights into how matrices behave.</p>
<section id="definition-of-the-trace" class="level4">
<h4 class="anchored" data-anchor-id="definition-of-the-trace">Definition of the Trace</h4>
<p>For a square matrix <span class="math inline">\(A = [a_{ij}]\)</span> of size <span class="math inline">\(n \times n\)</span>, the trace is the sum of the diagonal entries:</p>
<p><span class="math display">\[
\text{tr}(A) = a_{11} + a_{22} + \cdots + a_{nn}.
\]</span></p>
<p>Example:</p>
<p><span class="math display">\[
A = \begin{bmatrix}
2 &amp; 5 &amp; 7 \\
0 &amp; 3 &amp; 1 \\
4 &amp; 6 &amp; 8
\end{bmatrix},
\quad
\text{tr}(A) = 2 + 3 + 8 = 13.
\]</span></p>
<p>The trace extracts a single number summarizing the “diagonal content” of a matrix.</p>
</section>
<section id="properties-of-the-trace" class="level4">
<h4 class="anchored" data-anchor-id="properties-of-the-trace">Properties of the Trace</h4>
<p>The trace is linear and interacts nicely with multiplication and transposition:</p>
<ol type="1">
<li><p>Linearity:</p>
<ul>
<li><span class="math inline">\(\text{tr}(A + B) = \text{tr}(A) + \text{tr}(B)\)</span>.</li>
<li><span class="math inline">\(\text{tr}(cA) = c \cdot \text{tr}(A)\)</span>.</li>
</ul></li>
<li><p>Cyclic Property:</p>
<ul>
<li><span class="math inline">\(\text{tr}(AB) = \text{tr}(BA)\)</span>, as long as the products are defined.</li>
<li>More generally, <span class="math inline">\(\text{tr}(ABC) = \text{tr}(BCA) = \text{tr}(CAB)\)</span>.</li>
<li>But in general, <span class="math inline">\(\text{tr}(AB) \neq \text{tr}(A)\text{tr}(B)\)</span>.</li>
</ul></li>
<li><p>Transpose Invariance:</p>
<ul>
<li><span class="math inline">\(\text{tr}(A^T) = \text{tr}(A)\)</span>.</li>
</ul></li>
<li><p>Similarity Invariance:</p>
<ul>
<li>If <span class="math inline">\(B = P^{-1}AP\)</span>, then <span class="math inline">\(\text{tr}(B) = \text{tr}(A)\)</span>.</li>
<li>This means the trace is a similarity invariant, depending only on the linear transformation, not the basis.</li>
</ul></li>
</ol>
</section>
<section id="trace-and-eigenvalues" class="level4">
<h4 class="anchored" data-anchor-id="trace-and-eigenvalues">Trace and Eigenvalues</h4>
<p>One of the most important connections is between the trace and eigenvalues:</p>
<p><span class="math display">\[
\text{tr}(A) = \lambda_1 + \lambda_2 + \cdots + \lambda_n,
\]</span></p>
<p>where <span class="math inline">\(\lambda_i\)</span> are the eigenvalues of <span class="math inline">\(A\)</span> (counting multiplicity).</p>
<p>This links the simple diagonal sum to the deep spectral properties of the matrix.</p>
<p>Example:</p>
<p><span class="math display">\[
A = \begin{bmatrix} 1 &amp; 0 \\ 0 &amp; 3 \end{bmatrix}, \quad
\text{tr}(A) = 4, \quad
\lambda_1 = 1, \; \lambda_2 = 3, \quad \lambda_1 + \lambda_2 = 4.
\]</span></p>
</section>
<section id="other-basic-matrix-properties" class="level4">
<h4 class="anchored" data-anchor-id="other-basic-matrix-properties">Other Basic Matrix Properties</h4>
<p>Alongside the trace, here are some important algebraic facts that every student of linear algebra must know:</p>
<ol type="1">
<li><p>Determinant vs.&nbsp;Trace:</p>
<ul>
<li>For 2×2 matrices, <span class="math inline">\(A = \begin{bmatrix} a &amp; b \\ c &amp; d \end{bmatrix}\)</span>, <span class="math inline">\(\text{tr}(A) = a + d\)</span>, <span class="math inline">\(\det(A) = ad - bc\)</span>.</li>
<li>Together, trace and determinant encode the eigenvalues: roots of <span class="math inline">\(x^2 - \text{tr}(A)x + \det(A) = 0\)</span>.</li>
</ul></li>
<li><p>Norms and Inner Products:</p>
<ul>
<li>The Frobenius norm is defined using the trace: <span class="math inline">\(\|A\|_F = \sqrt{\text{tr}(A^TA)}\)</span>.</li>
</ul></li>
<li><p>Orthogonal Invariance:</p>
<ul>
<li>For any orthogonal matrix <span class="math inline">\(Q\)</span>, <span class="math inline">\(\text{tr}(Q^TAQ) = \text{tr}(A)\)</span>.</li>
</ul></li>
</ol>
</section>
<section id="geometric-and-practical-meaning" class="level4">
<h4 class="anchored" data-anchor-id="geometric-and-practical-meaning">Geometric and Practical Meaning</h4>
<ul>
<li>The trace of a transformation can be seen as the sum of its action along the coordinate axes.</li>
<li>In physics, the trace of the stress tensor measures pressure.</li>
<li>In probability, the trace of a covariance matrix is the total variance of a system.</li>
<li>In statistics and machine learning, the trace is often used as a measure of overall “size” or complexity of a model.</li>
</ul>
</section>
<section id="why-it-matters-14" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-14">Why It Matters</h4>
<p>The trace is deceptively simple but incredibly powerful:</p>
<ol type="1">
<li>It connects directly to eigenvalues, forming a bridge between raw matrix entries and spectral theory.</li>
<li>It is invariant under similarity, making it a reliable measure of a transformation independent of basis.</li>
<li>It shows up in optimization, physics, statistics, and quantum mechanics.</li>
<li>It simplifies computations: many proofs in linear algebra reduce to trace properties.</li>
</ol>
</section>
<section id="try-it-yourself-17" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-17">Try It Yourself</h4>
<ol type="1">
<li>Compute the trace of</li>
</ol>
<p><span class="math display">\[
\begin{bmatrix}
4 &amp; 2 &amp; 0 \\
-1 &amp; 3 &amp; 5 \\
7 &amp; 6 &amp; 1
\end{bmatrix}.
\]</span></p>
<ol start="2" type="1">
<li>Verify that <span class="math inline">\(\text{tr}(AB) = \text{tr}(BA)\)</span> for</li>
</ol>
<p><span class="math display">\[
A = \begin{bmatrix} 1 &amp; 2 \\ 0 &amp; 3 \end{bmatrix}, \quad
B = \begin{bmatrix} 4 &amp; 0 \\ 5 &amp; 6 \end{bmatrix}.
\]</span></p>
<ol start="3" type="1">
<li>For the 2×2 matrix</li>
</ol>
<p><span class="math display">\[
\begin{bmatrix} 2 &amp; 1 \\ 1 &amp; 2 \end{bmatrix},
\]</span></p>
<p>compute its eigenvalues and check that their sum equals the trace.</p>
<ol start="4" type="1">
<li>Challenge: Show that the total variance of a dataset with covariance matrix <span class="math inline">\(\Sigma\)</span> is equal to <span class="math inline">\(\text{tr}(\Sigma)\)</span>.</li>
</ol>
<p>Mastering the trace and its properties will prepare you for the next leap: understanding how matrices interact with volume, orientation, and determinants.</p>
</section>
</section>
<section id="affine-transforms-and-homogeneous-coordinates" class="level3">
<h3 class="anchored" data-anchor-id="affine-transforms-and-homogeneous-coordinates">19. Affine Transforms and Homogeneous Coordinates</h3>
<p>Up to now, matrices have been used to describe linear transformations: scaling, rotating, reflecting, projecting. But real-world geometry often involves more than just linear effects-it includes translations (shifts) as well. A pure linear map cannot move the origin, so to handle translations (and combinations of them with rotations, scalings, and shears), we extend our toolkit to affine transformations. The secret weapon that makes this work is the idea of homogeneous coordinates.</p>
<section id="what-is-an-affine-transformation" class="level4">
<h4 class="anchored" data-anchor-id="what-is-an-affine-transformation">What is an Affine Transformation?</h4>
<p>An affine transformation is any map of the form:</p>
<p><span class="math display">\[
f(\mathbf{x}) = A\mathbf{x} + \mathbf{b},
\]</span></p>
<p>where <span class="math inline">\(A\)</span> is a matrix (linear part) and <span class="math inline">\(\mathbf{b}\)</span> is a vector (translation part).</p>
<ul>
<li><span class="math inline">\(A\)</span> handles scaling, rotation, reflection, shear, or projection.</li>
<li><span class="math inline">\(\mathbf{b}\)</span> shifts everything by a constant amount.</li>
</ul>
<p>Examples in 2D:</p>
<ol type="1">
<li>Rotate by 90° and then shift right by 2.</li>
<li>Stretch vertically by 3 and shift upward by 1.</li>
</ol>
<p>Affine maps preserve parallel lines and ratios of distances along lines, but not necessarily angles or lengths.</p>
</section>
<section id="why-linear-maps-alone-arent-enough" class="level4">
<h4 class="anchored" data-anchor-id="why-linear-maps-alone-arent-enough">Why Linear Maps Alone Aren’t Enough</h4>
<p>If we only use a 2×2 matrix in 2D or 3×3 in 3D, the origin always stays fixed. That’s a limitation: real-world movements (like moving a shape from one place to another) require shifting the origin too. To capture both linear and translational effects uniformly, we need a clever trick.</p>
</section>
<section id="homogeneous-coordinates" class="level4">
<h4 class="anchored" data-anchor-id="homogeneous-coordinates">Homogeneous Coordinates</h4>
<p>The trick is to add one extra coordinate.</p>
<ul>
<li>In 2D, a point <span class="math inline">\((x, y)\)</span> becomes <span class="math inline">\((x, y, 1)\)</span>.</li>
<li>In 3D, a point <span class="math inline">\((x, y, z)\)</span> becomes <span class="math inline">\((x, y, z, 1)\)</span>.</li>
</ul>
<p>This new representation is called homogeneous coordinates. It allows us to fold translations into matrix multiplication.</p>
</section>
<section id="affine-transform-as-a-matrix-in-homogeneous-form" class="level4">
<h4 class="anchored" data-anchor-id="affine-transform-as-a-matrix-in-homogeneous-form">Affine Transform as a Matrix in Homogeneous Form</h4>
<p>In 2D:</p>
<p><span class="math display">\[
\begin{bmatrix}
a &amp; b &amp; t_x \\
c &amp; d &amp; t_y \\
0 &amp; 0 &amp; 1
\end{bmatrix}
\begin{bmatrix}
x \\ y \\ 1
\end{bmatrix}
=
\begin{bmatrix}
ax + by + t_x \\
cx + dy + t_y \\
1
\end{bmatrix}.
\]</span></p>
<p>Here,</p>
<ul>
<li>The 2×2 block <span class="math inline">\(\begin{bmatrix} a &amp; b \\ c &amp; d \end{bmatrix}\)</span> is the linear part.</li>
<li>The last column <span class="math inline">\(\begin{bmatrix} t_x \\ t_y \end{bmatrix}\)</span> is the translation.</li>
</ul>
<p>So with one unified matrix, we can handle both linear transformations and shifts.</p>
</section>
<section id="examples-in-2d" class="level4">
<h4 class="anchored" data-anchor-id="examples-in-2d">Examples in 2D</h4>
<ol type="1">
<li>Translation by (2, 3):</li>
</ol>
<p><span class="math display">\[
\begin{bmatrix}
1 &amp; 0 &amp; 2 \\
0 &amp; 1 &amp; 3 \\
0 &amp; 0 &amp; 1
\end{bmatrix}.
\]</span></p>
<ol start="2" type="1">
<li>Scaling by 2 in x and 3 in y, then shifting by (–1, 4):</li>
</ol>
<p><span class="math display">\[
\begin{bmatrix}
2 &amp; 0 &amp; -1 \\
0 &amp; 3 &amp; 4 \\
0 &amp; 0 &amp; 1
\end{bmatrix}.
\]</span></p>
<ol start="3" type="1">
<li>Rotation by 90° and shift right by 5:</li>
</ol>
<p><span class="math display">\[
\begin{bmatrix}
0 &amp; -1 &amp; 5 \\
1 &amp; 0 &amp; 0 \\
0 &amp; 0 &amp; 1
\end{bmatrix}.
\]</span></p>
</section>
<section id="homogeneous-coordinates-in-3d" class="level4">
<h4 class="anchored" data-anchor-id="homogeneous-coordinates-in-3d">Homogeneous Coordinates in 3D</h4>
<p>In 3D, affine transformations use 4×4 matrices. The upper-left 3×3 block handles rotation, scaling, or shear; the last column encodes translation.</p>
<p>Example: translation by (2, –1, 4):</p>
<p><span class="math display">\[
\begin{bmatrix}
1 &amp; 0 &amp; 0 &amp; 2 \\
0 &amp; 1 &amp; 0 &amp; -1 \\
0 &amp; 0 &amp; 1 &amp; 4 \\
0 &amp; 0 &amp; 0 &amp; 1
\end{bmatrix}.
\]</span></p>
<p>This formulation is universal in computer graphics and robotics.</p>
</section>
<section id="why-it-matters-15" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-15">Why It Matters</h4>
<ol type="1">
<li>Unified representation: Using homogeneous coordinates, we can treat translations as matrices, enabling consistent matrix multiplication for all transformations.</li>
<li>Practicality: This approach underpins 3D graphics pipelines, animation, CAD, robotics, and computer vision.</li>
<li>Composability: Multiple affine transformations can be combined into a single homogeneous matrix by multiplying them.</li>
<li>Geometry preserved: Affine maps preserve straight lines and parallelism, essential in engineering and design.</li>
</ol>
</section>
<section id="try-it-yourself-18" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-18">Try It Yourself</h4>
<ol type="1">
<li>Write the homogeneous matrix that reflects across the x-axis and then shifts up by 3. Apply it to <span class="math inline">\((2, 1)\)</span>.</li>
<li>Construct a 4×4 homogeneous matrix that rotates around the z-axis by 90° and translates by (1, 2, 0).</li>
<li>Show that multiplying two 3×3 homogeneous matrices in 2D yields another valid affine transform.</li>
<li>Challenge: Prove that affine maps preserve parallel lines by applying a general affine matrix to two parallel lines and checking their slopes.</li>
</ol>
<p>Mastering affine transformations and homogeneous coordinates bridges the gap between pure linear algebra and real-world geometry, giving you the mathematical foundation behind computer graphics, robotics, and spatial modeling.</p>
</section>
</section>
<section id="computing-with-matrices-cost-counts-and-simple-speedups" class="level3">
<h3 class="anchored" data-anchor-id="computing-with-matrices-cost-counts-and-simple-speedups">20. Computing with Matrices (Cost Counts and Simple Speedups)</h3>
<p>Thus far, we have studied what matrices are and what they represent. But in practice, working with matrices also means thinking about computation-how much work operations take, how algorithms can be sped up, and why structure matters. This section introduces the basic ideas of computational cost in matrix operations, simple strategies for efficiency, and why these considerations are crucial in modern applications.</p>
<section id="counting-operations-the-cost-model" class="level4">
<h4 class="anchored" data-anchor-id="counting-operations-the-cost-model">Counting Operations: The Cost Model</h4>
<p>The simplest way to measure the cost of a matrix operation is to count the basic arithmetic operations (additions and multiplications).</p>
<ul>
<li><p>Matrix–vector product: For an <span class="math inline">\(m \times n\)</span> matrix and an <span class="math inline">\(n \times 1\)</span> vector:</p>
<ul>
<li>Each of the <span class="math inline">\(m\)</span> output entries requires <span class="math inline">\(n\)</span> multiplications and <span class="math inline">\(n-1\)</span> additions.</li>
<li>Total cost ≈ <span class="math inline">\(2mn\)</span> operations.</li>
</ul></li>
<li><p>Matrix–matrix product: For an <span class="math inline">\(m \times n\)</span> matrix times an <span class="math inline">\(n \times p\)</span> matrix:</p>
<ul>
<li>Each of the <span class="math inline">\(mp\)</span> entries requires <span class="math inline">\(n\)</span> multiplications and <span class="math inline">\(n-1\)</span> additions.</li>
<li>Total cost ≈ <span class="math inline">\(2mnp\)</span> operations.</li>
</ul></li>
<li><p>Gaussian elimination (solving <span class="math inline">\(Ax=b\)</span>): For an <span class="math inline">\(n \times n\)</span> system:</p>
<ul>
<li>Roughly <span class="math inline">\(\tfrac{2}{3}n^3\)</span> operations.</li>
</ul></li>
</ul>
<p>These counts show how quickly costs grow with dimension. Doubling <span class="math inline">\(n\)</span> makes the work 8 times larger for elimination.</p>
</section>
<section id="why-cost-counts-matter" class="level4">
<h4 class="anchored" data-anchor-id="why-cost-counts-matter">Why Cost Counts Matter</h4>
<ol type="1">
<li>Scalability: Small problems (2×2 or 3×3) are trivial, but modern datasets involve matrices with millions of rows. Knowing the cost is essential.</li>
<li>Feasibility: Some exact algorithms become impossible for very large matrices. Approximation methods are used instead.</li>
<li>Optimization: Engineers and scientists design specialized algorithms to reduce costs by exploiting structure (sparsity, symmetry, triangular form).</li>
</ol>
</section>
<section id="simple-speedups-with-structure" class="level4">
<h4 class="anchored" data-anchor-id="simple-speedups-with-structure">Simple Speedups with Structure</h4>
<ul>
<li>Diagonal Matrices: Multiplying by a diagonal matrix costs only <span class="math inline">\(n\)</span> operations (scale each component).</li>
<li>Triangular Matrices: Solving triangular systems requires only <span class="math inline">\(\tfrac{1}{2}n^2\)</span> operations (substitution), far cheaper than general elimination.</li>
<li>Sparse Matrices: If most entries are zero, we skip multiplications by zero. For large sparse systems, cost scales with the number of nonzeros, not <span class="math inline">\(n^2\)</span>.</li>
<li>Block Matrices: Breaking matrices into blocks allows algorithms to reuse optimized small-matrix routines (common in BLAS libraries).</li>
</ul>
</section>
<section id="memory-considerations" class="level4">
<h4 class="anchored" data-anchor-id="memory-considerations">Memory Considerations</h4>
<p>Cost is not only arithmetic: storage also matters.</p>
<ul>
<li>A dense <span class="math inline">\(n \times n\)</span> matrix requires <span class="math inline">\(n^2\)</span> entries of memory.</li>
<li>Sparse storage formats (like CSR, COO) record only nonzero entries and their positions, saving massive space.</li>
<li>Memory access speed can dominate arithmetic cost in large computations.</li>
</ul>
</section>
<section id="parallelism-and-hardware" class="level4">
<h4 class="anchored" data-anchor-id="parallelism-and-hardware">Parallelism and Hardware</h4>
<p>Modern computing leverages hardware for speed:</p>
<ul>
<li>Vectorization (SIMD): Perform many multiplications at once.</li>
<li>Parallelization: Split work across many CPU cores.</li>
<li>GPUs: Specialize in massive parallel matrix–vector and matrix–matrix operations (critical in deep learning).</li>
</ul>
<p>This is why linear algebra libraries (BLAS, LAPACK, cuBLAS) are indispensable: they squeeze performance from hardware.</p>
</section>
<section id="why-it-matters-16" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-16">Why It Matters</h4>
<ol type="1">
<li>Efficiency: Understanding cost lets us choose the right algorithm for the job.</li>
<li>Algorithm design: Structured matrices (diagonal, sparse, orthogonal) make computations much faster and more stable.</li>
<li>Applications: Every field that uses matrices-graphics, optimization, statistics, AI-relies on efficient computation.</li>
<li>Foundations: Later topics like LU/QR/SVD factorization are motivated by balancing cost and stability.</li>
</ol>
</section>
<section id="try-it-yourself-19" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-19">Try It Yourself</h4>
<ol type="1">
<li>Compute the number of operations required for multiplying a 1000×500 matrix with a 500×200 matrix. Compare with multiplying a 1000×1000 dense matrix by a vector.</li>
<li>Show how solving a 3×3 triangular system is faster than Gaussian elimination. Count the exact multiplications and additions.</li>
<li>Construct a sparse 5×5 matrix with only 7 nonzero entries. Estimate the cost of multiplying it by a vector versus a dense 5×5 matrix.</li>
<li>Challenge: Suppose you need to store a 1,000,000×1,000,000 dense matrix. Estimate how much memory (in bytes) it would take if each entry is 8 bytes. Could it fit on a laptop? Why do sparse formats save the day?</li>
</ol>
<p>By learning to count costs and exploit structure, you prepare yourself not only to understand matrices abstractly but also to use them effectively in real-world, large-scale problems. This balance between theory and computation is at the heart of modern linear algebra.</p>
</section>
<section id="closing-1" class="level4">
<h4 class="anchored" data-anchor-id="closing-1">Closing</h4>
<pre><code>Patterns intertwine,
transformations gently fold,
structure in the square.</code></pre>
</section>
</section>
</section>
<section id="chapter-3.-linear-systems-and-elimination" class="level2">
<h2 class="anchored" data-anchor-id="chapter-3.-linear-systems-and-elimination">Chapter 3. Linear Systems and Elimination</h2>
<section id="from-equations-to-matrices" class="level3">
<h3 class="anchored" data-anchor-id="from-equations-to-matrices">21. From Equations to Matrices</h3>
<p>Linear algebra often begins with systems of equations-collections of unknowns linked by linear relationships. While these systems can be solved directly using substitution or elimination, they quickly become messy when there are many variables. The key insight of linear algebra is that all systems of linear equations can be captured compactly by matrices and vectors. This section explains how we move from equations written out in words and symbols to the matrix form that powers computation.</p>
<section id="a-simple-example" class="level4">
<h4 class="anchored" data-anchor-id="a-simple-example">A Simple Example</h4>
<p>Consider this system of two equations in two unknowns:</p>
<p><span class="math display">\[
\begin{cases}  
2x + y = 5 \\  
3x - y = 4  
\end{cases}
\]</span></p>
<p>At first glance, this is just algebra: two equations, two unknowns. But notice the structure: each equation is a sum of multiples of the variables, set equal to a constant. This pattern-linear combinations of unknowns equal to a result-is exactly what matrices capture.</p>
</section>
<section id="writing-in-coefficient-table-form" class="level4">
<h4 class="anchored" data-anchor-id="writing-in-coefficient-table-form">Writing in Coefficient Table Form</h4>
<p>Extract the coefficients of each variable from the system:</p>
<ul>
<li>First equation: coefficients are <span class="math inline">\(2\)</span> for <span class="math inline">\(x\)</span>, <span class="math inline">\(1\)</span> for <span class="math inline">\(y\)</span>.</li>
<li>Second equation: coefficients are <span class="math inline">\(3\)</span> for <span class="math inline">\(x\)</span>, <span class="math inline">\(-1\)</span> for <span class="math inline">\(y\)</span>.</li>
</ul>
<p>Arrange these coefficients in a rectangular array:</p>
<p><span class="math display">\[
A = \begin{bmatrix}  
2 &amp; 1 \\  
3 &amp; -1  
\end{bmatrix}.
\]</span></p>
<p>This matrix <span class="math inline">\(A\)</span> is called the coefficient matrix.</p>
<p>Next, write the unknowns as a vector:</p>
<p><span class="math display">\[
\mathbf{x} = \begin{bmatrix} x \\ y \end{bmatrix}.
\]</span></p>
<p>Finally, write the right-hand side constants as another vector:</p>
<p><span class="math display">\[
\mathbf{b} = \begin{bmatrix} 5 \\ 4 \end{bmatrix}.
\]</span></p>
<p>Now the entire system can be written in a single line:</p>
<p><span class="math display">\[
A\mathbf{x} = \mathbf{b}.
\]</span></p>
</section>
<section id="why-this-is-powerful" class="level4">
<h4 class="anchored" data-anchor-id="why-this-is-powerful">Why This is Powerful</h4>
<p>This compact form hides no information; it is equivalent to the original equations. But it gives us enormous advantages:</p>
<ol type="1">
<li>Clarity: We see the structure clearly-the system is “matrix times vector equals vector.”</li>
<li>Scalability: Whether we have 2 equations or 2000, the same notation applies.</li>
<li>Tools: All the machinery of matrix operations (elimination, inverses, decompositions) now becomes available.</li>
<li>Geometry: The matrix equation <span class="math inline">\(A\mathbf{x} = \mathbf{b}\)</span> means: combine the columns of <span class="math inline">\(A\)</span> (scaled by entries of x) to land on b.</li>
</ol>
</section>
<section id="a-larger-example" class="level4">
<h4 class="anchored" data-anchor-id="a-larger-example">A Larger Example</h4>
<p>System of three equations in three unknowns:</p>
<p><span class="math display">\[
\begin{cases}  
x + 2y - z = 2 \\  
2x - y + 3z = 1 \\  
3x + y + 2z = 4  
\end{cases}
\]</span></p>
<ul>
<li><p>Coefficient matrix:</p>
<p><span class="math display">\[
A = \begin{bmatrix}  
1 &amp; 2 &amp; -1 \\  
2 &amp; -1 &amp; 3 \\  
3 &amp; 1 &amp; 2  
\end{bmatrix}.
\]</span></p></li>
<li><p>Unknown vector:</p>
<p><span class="math display">\[
\mathbf{x} = \begin{bmatrix} x \\ y \\ z \end{bmatrix}.
\]</span></p></li>
<li><p>Constant vector:</p>
<p><span class="math display">\[
\mathbf{b} = \begin{bmatrix} 2 \\ 1 \\ 4 \end{bmatrix}.
\]</span></p></li>
</ul>
<p>Matrix form:</p>
<p><span class="math display">\[
A\mathbf{x} = \mathbf{b}.
\]</span></p>
<p>This single equation captures three equations and three unknowns in one object.</p>
</section>
<section id="row-vs.-column-view" class="level4">
<h4 class="anchored" data-anchor-id="row-vs.-column-view">Row vs.&nbsp;Column View</h4>
<ul>
<li>Row view: Each row of <span class="math inline">\(A\)</span> dotted with x gives one equation.</li>
<li>Column view: The entire system means b is a linear combination of the columns of <span class="math inline">\(A\)</span>.</li>
</ul>
<p>For the 2×2 case earlier:</p>
<p><span class="math display">\[
A\mathbf{x} = \begin{bmatrix} 2 &amp; 1 \\ 3 &amp; -1 \end{bmatrix}  
\begin{bmatrix} x \\ y \end{bmatrix}  
= x \begin{bmatrix} 2 \\ 3 \end{bmatrix} + y \begin{bmatrix} 1 \\ -1 \end{bmatrix}.
\]</span></p>
<p>So solving the system means finding scalars <span class="math inline">\(x, y\)</span> that combine the columns of <span class="math inline">\(A\)</span> to reach <span class="math inline">\(\mathbf{b}\)</span>.</p>
</section>
<section id="augmented-matrix-form" class="level4">
<h4 class="anchored" data-anchor-id="augmented-matrix-form">Augmented Matrix Form</h4>
<p>Sometimes we want to save space further. We can put the coefficients and constants side by side in an augmented matrix:</p>
<p><span class="math display">\[
[A | \mathbf{b}] =  
\begin{bmatrix}  
2 &amp; 1 &amp; | &amp; 5 \\  
3 &amp; -1 &amp; | &amp; 4  
\end{bmatrix}.
\]</span></p>
<p>This form is especially useful for elimination methods, where we manipulate rows without writing variables at each step.</p>
</section>
<section id="why-it-matters-17" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-17">Why It Matters</h4>
<p>This step-rewriting equations as matrix form-is the gateway into linear algebra. Once you can do it, you no longer think of systems of equations as isolated lines on paper, but as a unified object that can be studied with general tools. It opens the door to:</p>
<ul>
<li>Gaussian elimination,</li>
<li>rank and null space,</li>
<li>determinants,</li>
<li>eigenvalues,</li>
<li>optimization methods.</li>
</ul>
<p>Every major idea flows from this compact representation.</p>
</section>
<section id="try-it-yourself-20" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-20">Try It Yourself</h4>
<ol type="1">
<li><p>Write the system</p>
<p><span class="math display">\[
\begin{cases}  
4x - y = 7 \\  
-2x + 3y = 5  
\end{cases}
\]</span></p>
<p>in matrix form.</p></li>
<li><p>For the system</p>
<p><span class="math display">\[
\begin{cases}  
x + y + z = 6 \\  
2x - y + z = 3 \\  
x - y - z = -2  
\end{cases}
\]</span></p>
<p>build the coefficient matrix, unknown vector, and constant vector.</p></li>
<li><p>Express the augmented matrix for the above system.</p></li>
<li><p>Challenge: Interpret the system in column view. What does it mean geometrically to express <span class="math inline">\((6, 3, -2)\)</span> as a linear combination of the columns of the coefficient matrix?</p></li>
</ol>
<p>By practicing these rewrites, you will see that linear algebra is not about juggling many equations-it is about seeing structure in one compact equation. This step transforms scattered equations into the language of matrices, where the real power begins.</p>
</section>
</section>
<section id="row-operations" class="level3">
<h3 class="anchored" data-anchor-id="row-operations">22. Row Operations</h3>
<p>Once a system of linear equations has been expressed as a matrix, the next step is to simplify that matrix into a form where the solutions become clear. The main tool for this simplification is the set of elementary row operations. These operations allow us to manipulate the rows of a matrix in systematic ways that preserve the solution set of the corresponding system of equations.</p>
<section id="the-three-types-of-row-operations" class="level4">
<h4 class="anchored" data-anchor-id="the-three-types-of-row-operations">The Three Types of Row Operations</h4>
<p>There are exactly three types of legal row operations, each with a clear algebraic meaning:</p>
<ol type="1">
<li><p>Row Swapping (<span class="math inline">\(R_i \leftrightarrow R_j\)</span>): Exchange two rows. This corresponds to reordering equations in a system. Since the order of equations doesn’t change the solutions, this operation is always valid.</p>
<p>Example:</p>
<p><span class="math display">\[
\begin{bmatrix}  
2 &amp; 1 &amp; | &amp; 5 \\  
3 &amp; -1 &amp; | &amp; 4  
\end{bmatrix}  
\quad \longrightarrow \quad  
\begin{bmatrix}  
3 &amp; -1 &amp; | &amp; 4 \\  
2 &amp; 1 &amp; | &amp; 5  
\end{bmatrix}.
\]</span></p></li>
<li><p>Row Scaling (<span class="math inline">\(R_i \to cR_i, \; c \neq 0\)</span>): Multiply all entries in a row by a nonzero constant. This is like multiplying both sides of an equation by the same number, which doesn’t change its truth.</p>
<p>Example:</p>
<p><span class="math display">\[
\begin{bmatrix}  
2 &amp; 1 &amp; | &amp; 5 \\  
3 &amp; -1 &amp; | &amp; 4  
\end{bmatrix}  
\quad \longrightarrow \quad  
\begin{bmatrix}  
1 &amp; \tfrac{1}{2} &amp; | &amp; \tfrac{5}{2} \\  
3 &amp; -1 &amp; | &amp; 4  
\end{bmatrix}.
\]</span></p></li>
<li><p>Row Replacement (<span class="math inline">\(R_i \to R_i + cR_j\)</span>): Add a multiple of one row to another. This corresponds to replacing one equation with a linear combination of itself and another, a fundamental elimination step.</p>
<p>Example:</p>
<p><span class="math display">\[
\begin{bmatrix}  
2 &amp; 1 &amp; | &amp; 5 \\  
3 &amp; -1 &amp; | &amp; 4  
\end{bmatrix}  
\quad \overset{R_2 \to R_2 - \tfrac{3}{2}R_1}{\longrightarrow} \quad  
\begin{bmatrix}  
2 &amp; 1 &amp; | &amp; 5 \\  
0 &amp; -\tfrac{5}{2} &amp; | &amp; -\tfrac{7}{2}  
\end{bmatrix}.
\]</span></p></li>
</ol>
</section>
<section id="why-these-are-the-only-allowed-operations" class="level4">
<h4 class="anchored" data-anchor-id="why-these-are-the-only-allowed-operations">Why These Are the Only Allowed Operations</h4>
<p>These three operations are the backbone of elimination because they do not alter the solution set of the system. Each is equivalent to applying an invertible transformation:</p>
<ul>
<li>Row swaps are reversible (swap back).</li>
<li>Row scalings by <span class="math inline">\(c\)</span> can be undone by scaling by <span class="math inline">\(1/c\)</span>.</li>
<li>Row replacements can be undone by adding the opposite multiple.</li>
</ul>
<p>Thus, each operation is invertible, and the transformed system is always equivalent to the original.</p>
</section>
<section id="row-operations-as-matrices" class="level4">
<h4 class="anchored" data-anchor-id="row-operations-as-matrices">Row Operations as Matrices</h4>
<p>Each elementary row operation can itself be represented by multiplying on the left with a special matrix called an elementary matrix.</p>
<p>For example:</p>
<ul>
<li><p>Swapping rows 1 and 2 in a 2×2 system is done by</p>
<p><span class="math display">\[
E = \begin{bmatrix} 0 &amp; 1 \\ 1 &amp; 0 \end{bmatrix}.
\]</span></p></li>
<li><p>Scaling row 1 by 3 in a 2×2 system is done by</p>
<p><span class="math display">\[
E = \begin{bmatrix} 3 &amp; 0 \\ 0 &amp; 1 \end{bmatrix}.
\]</span></p></li>
</ul>
<p>This perspective is crucial later for factorization methods like LU decomposition, where elimination is expressed as a product of elementary matrices.</p>
</section>
<section id="step-by-step-example" class="level4">
<h4 class="anchored" data-anchor-id="step-by-step-example">Step-by-Step Example</h4>
<p>System:</p>
<p><span class="math display">\[
\begin{cases}  
x + 2y = 4 \\  
3x + 4y = 10  
\end{cases}
\]</span></p>
<p>Augmented matrix:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; | &amp; 4 \\  
3 &amp; 4 &amp; | &amp; 10  
\end{bmatrix}.
\]</span></p>
<ol type="1">
<li><p>Eliminate the <span class="math inline">\(3x\)</span> under the first pivot: <span class="math inline">\(R_2 \to R_2 - 3R_1\)</span>.</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; | &amp; 4 \\  
0 &amp; -2 &amp; | &amp; -2  
\end{bmatrix}.
\]</span></p></li>
<li><p>Scale the second row: <span class="math inline">\(R_2 \to -\tfrac{1}{2}R_2\)</span>.</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; | &amp; 4 \\  
0 &amp; 1 &amp; | &amp; 1  
\end{bmatrix}.
\]</span></p></li>
<li><p>Eliminate above the pivot: <span class="math inline">\(R_1 \to R_1 - 2R_2\)</span>.</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 0 &amp; | &amp; 2 \\  
0 &amp; 1 &amp; | &amp; 1  
\end{bmatrix}.
\]</span></p></li>
</ol>
<p>Solution: <span class="math inline">\(x = 2, \; y = 1\)</span>.</p>
</section>
<section id="geometry-of-row-operations" class="level4">
<h4 class="anchored" data-anchor-id="geometry-of-row-operations">Geometry of Row Operations</h4>
<p>Row operations do not alter the solution space:</p>
<ul>
<li>Swapping rows reorders equations but keeps the same lines or planes.</li>
<li>Scaling rows rescales equations but leaves their geometric set unchanged.</li>
<li>Adding rows corresponds to combining constraints, but the shared intersection (solution set) is preserved.</li>
</ul>
<p>Thus, row operations act like “reshaping the system” while leaving the intersection intact.</p>
</section>
<section id="why-it-matters-18" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-18">Why It Matters</h4>
<p>Row operations are the essential moves in solving linear systems by hand or computer. They:</p>
<ol type="1">
<li>Make elimination systematic.</li>
<li>Preserve solution sets while simplifying structure.</li>
<li>Lay the groundwork for echelon forms, rank, and factorization.</li>
<li>Provide the mechanical steps that computers automate in Gaussian elimination.</li>
</ol>
</section>
<section id="try-it-yourself-21" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-21">Try It Yourself</h4>
<ol type="1">
<li><p>Apply row operations to reduce</p>
<p><span class="math display">\[
\begin{bmatrix}  
2 &amp; 1 &amp; | &amp; 7 \\  
1 &amp; -1 &amp; | &amp; 1  
\end{bmatrix}
\]</span></p>
<p>to a form where the solution is obvious.</p></li>
<li><p>Show explicitly why swapping two equations in a system doesn’t change its solutions.</p></li>
<li><p>Construct the elementary matrix for “add –2 times row 1 to row 3” in a 3×3 system.</p></li>
<li><p>Challenge: Prove that any elementary row operation corresponds to multiplication by an invertible matrix.</p></li>
</ol>
<p>Mastering these operations equips you with the mechanical and conceptual foundation for the next stage: systematically reducing matrices to row-echelon form.</p>
</section>
</section>
<section id="row-echelon-and-reduced-row-echelon-forms" class="level3">
<h3 class="anchored" data-anchor-id="row-echelon-and-reduced-row-echelon-forms">23. Row-Echelon and Reduced Row-Echelon Forms</h3>
<p>After introducing row operations, the natural question is: <em>what are we trying to achieve by performing them?</em> The answer is to transform a matrix into a standardized, simplified form where the solutions to the corresponding system of equations can be read off directly. Two such standardized forms are central in linear algebra: row-echelon form (REF) and reduced row-echelon form (RREF).</p>
<section id="row-echelon-form-ref" class="level4">
<h4 class="anchored" data-anchor-id="row-echelon-form-ref">Row-Echelon Form (REF)</h4>
<p>A matrix is in row-echelon form if:</p>
<ol type="1">
<li>All nonzero rows are above any rows of all zeros.</li>
<li>In each nonzero row, the first nonzero entry (called the leading entry or pivot) is to the right of the leading entry of the row above it.</li>
<li>All entries below a pivot are zero.</li>
</ol>
<p>Example of REF:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; 3 &amp; | &amp; 4 \\  
0 &amp; 1 &amp; -1 &amp; | &amp; 2 \\  
0 &amp; 0 &amp; 5 &amp; | &amp; -3 \\  
0 &amp; 0 &amp; 0 &amp; | &amp; 0  
\end{bmatrix}.
\]</span></p>
<p>Here, the pivots are the first 1 in row 1, the 1 in row 2, and the 5 in row 3. Each pivot is to the right of the one above it, and all entries below pivots are zero.</p>
</section>
<section id="reduced-row-echelon-form-rref" class="level4">
<h4 class="anchored" data-anchor-id="reduced-row-echelon-form-rref">Reduced Row-Echelon Form (RREF)</h4>
<p>A matrix is in reduced row-echelon form if, in addition to the rules of REF:</p>
<ol type="1">
<li>Each pivot is equal to 1.</li>
<li>Each pivot is the only nonzero entry in its column (everything above and below pivots is zero).</li>
</ol>
<p>Example of RREF:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 0 &amp; 0 &amp; | &amp; 3 \\  
0 &amp; 1 &amp; 0 &amp; | &amp; -2 \\  
0 &amp; 0 &amp; 1 &amp; | &amp; 1  
\end{bmatrix}.
\]</span></p>
<p>This form is so simplified that solutions can be read directly: here, <span class="math inline">\(x=3\)</span>, <span class="math inline">\(y=-2\)</span>, <span class="math inline">\(z=1\)</span>.</p>
</section>
<section id="relationship-between-ref-and-rref" class="level4">
<h4 class="anchored" data-anchor-id="relationship-between-ref-and-rref">Relationship Between REF and RREF</h4>
<ul>
<li>REF is easier to reach-it only requires eliminating entries below pivots.</li>
<li>RREF requires going further-clearing entries above pivots and scaling pivots to 1.</li>
<li>Every matrix can be reduced to REF (many possible versions), but RREF is unique: no matter how you proceed, if you carry out all row operations fully, you end with the same RREF.</li>
</ul>
</section>
<section id="example-step-by-step-to-rref" class="level4">
<h4 class="anchored" data-anchor-id="example-step-by-step-to-rref">Example: Step-by-Step to RREF</h4>
<p>System:</p>
<p><span class="math display">\[
\begin{cases}  
x + 2y + z = 4 \\  
2x + 5y + z = 7 \\  
3x + 6y + 2z = 10  
\end{cases}
\]</span></p>
<p>Augmented matrix:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; 1 &amp; | &amp; 4 \\  
2 &amp; 5 &amp; 1 &amp; | &amp; 7 \\  
3 &amp; 6 &amp; 2 &amp; | &amp; 10  
\end{bmatrix}.
\]</span></p>
<ol type="1">
<li><p>Eliminate below first pivot (the 1 in row 1, col 1):</p>
<ul>
<li><span class="math inline">\(R_2 \to R_2 - 2R_1\)</span></li>
<li><span class="math inline">\(R_3 \to R_3 - 3R_1\)</span></li>
</ul>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; 1 &amp; | &amp; 4 \\  
0 &amp; 1 &amp; -1 &amp; | &amp; -1 \\  
0 &amp; 0 &amp; -1 &amp; | &amp; -2  
\end{bmatrix}.
\]</span></p>
<p>This is now in REF.</p></li>
<li><p>Scale pivots and eliminate above them:</p>
<ul>
<li><span class="math inline">\(R_3 \to -R_3\)</span> to make pivot 1.</li>
<li><span class="math inline">\(R_2 \to R_2 + R_3\)</span>.</li>
<li><span class="math inline">\(R_1 \to R_1 - R_2 - R_3\)</span>.</li>
</ul>
<p>Final:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 0 &amp; 0 &amp; | &amp; 2 \\  
0 &amp; 1 &amp; 0 &amp; | &amp; 1 \\  
0 &amp; 0 &amp; 1 &amp; | &amp; 2  
\end{bmatrix}.
\]</span></p></li>
</ol>
<p>Solution: <span class="math inline">\(x=2, y=1, z=2\)</span>.</p>
</section>
<section id="geometry-of-ref-and-rref" class="level4">
<h4 class="anchored" data-anchor-id="geometry-of-ref-and-rref">Geometry of REF and RREF</h4>
<ul>
<li>REF corresponds to simplifying the system step by step, making it “triangular” so variables can be solved one after another.</li>
<li>RREF corresponds to a system that is fully disentangled-each variable isolated, with its value or free-variable relationship explicitly visible.</li>
</ul>
</section>
<section id="why-it-matters-19" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-19">Why It Matters</h4>
<ol type="1">
<li>REF is the foundation of Gaussian elimination, the workhorse algorithm for solving systems.</li>
<li>RREF gives complete clarity: unique representation of solution sets, revealing free and pivot variables.</li>
<li>RREF underlies algorithms in computer algebra systems, symbolic solvers, and educational tools.</li>
<li>Understanding these forms builds intuition for rank, null space, and solution structure.</li>
</ol>
</section>
<section id="try-it-yourself-22" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-22">Try It Yourself</h4>
<ol type="1">
<li><p>Reduce</p>
<p><span class="math display">\[
\begin{bmatrix}  
2 &amp; 4 &amp; | &amp; 6 \\  
1 &amp; 3 &amp; | &amp; 5  
\end{bmatrix}
\]</span></p>
<p>to REF, then RREF.</p></li>
<li><p>Find the RREF of</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 1 &amp; 1 &amp; | &amp; 3 \\  
2 &amp; 3 &amp; 4 &amp; | &amp; 8 \\  
1 &amp; 2 &amp; 3 &amp; | &amp; 5  
\end{bmatrix}.
\]</span></p></li>
<li><p>Explain why two different elimination sequences can lead to different REF but the same RREF.</p></li>
<li><p>Challenge: Prove that every matrix has a unique RREF by considering the effect of row operations systematically.</p></li>
</ol>
<p>Reaching row-echelon and reduced row-echelon forms transforms messy systems into structured ones, turning algebraic clutter into an organized path to solutions.</p>
</section>
</section>
<section id="pivots-free-variables-and-leading-ones" class="level3">
<h3 class="anchored" data-anchor-id="pivots-free-variables-and-leading-ones">24. Pivots, Free Variables, and Leading Ones</h3>
<p>When reducing a matrix to row-echelon or reduced row-echelon form, certain positions in the matrix take on a special importance. These are the pivots-the leading nonzero entries in each row. Around them, the entire solution structure of a linear system is organized. Understanding pivots, the variables they anchor, and the freedom that arises from non-pivot columns is essential to solving linear equations systematically.</p>
<section id="what-is-a-pivot" class="level4">
<h4 class="anchored" data-anchor-id="what-is-a-pivot">What is a Pivot?</h4>
<p>In row-echelon form, a pivot is the first nonzero entry in a row, moving from left to right. After scaling in reduced row-echelon form, each pivot is set to exactly 1.</p>
<p>Example:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; 0 &amp; | &amp; 5 \\  
0 &amp; 1 &amp; 3 &amp; | &amp; -2 \\  
0 &amp; 0 &amp; 0 &amp; | &amp; 0  
\end{bmatrix}
\]</span></p>
<ul>
<li>Pivot in row 1: the 1 in column 1.</li>
<li>Pivot in row 2: the 1 in column 2.</li>
<li>Column 3 has no pivot.</li>
</ul>
<p>Columns with pivots are pivot columns. Columns without pivots correspond to free variables.</p>
</section>
<section id="pivot-variables-vs.-free-variables" class="level4">
<h4 class="anchored" data-anchor-id="pivot-variables-vs.-free-variables">Pivot Variables vs.&nbsp;Free Variables</h4>
<ul>
<li>Pivot variables: Variables that align with pivot columns. They are determined by the equations.</li>
<li>Free variables: Variables that align with non-pivot columns. They are unconstrained and can take arbitrary values.</li>
</ul>
<p>Example:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 0 &amp; 2 &amp; | &amp; 3 \\  
0 &amp; 1 &amp; -1 &amp; | &amp; 4  
\end{bmatrix}.
\]</span></p>
<p>This corresponds to:</p>
<p><span class="math display">\[
x_1 + 2x_3 = 3, \quad x_2 - x_3 = 4.
\]</span></p>
<p>Here:</p>
<ul>
<li><span class="math inline">\(x_1\)</span> and <span class="math inline">\(x_2\)</span> are pivot variables (from pivot columns 1 and 2).</li>
<li><span class="math inline">\(x_3\)</span> is a free variable.</li>
</ul>
<p>Thus, <span class="math inline">\(x_1\)</span> and <span class="math inline">\(x_2\)</span> depend on <span class="math inline">\(x_3\)</span>:</p>
<p><span class="math display">\[
x_1 = 3 - 2x_3, \quad x_2 = 4 + x_3, \quad x_3 \text{ free}.
\]</span></p>
<p>The solution set is infinite, described by the freedom in <span class="math inline">\(x_3\)</span>.</p>
</section>
<section id="geometric-meaning-1" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-1">Geometric Meaning</h4>
<ul>
<li>Pivot variables represent coordinates that are “pinned down.”</li>
<li>Free variables correspond to directions along which the solution can extend infinitely.</li>
</ul>
<p>In 2D:</p>
<ul>
<li>If there is one pivot variable and one free variable, solutions form a line. In 3D:</li>
<li>Two pivots, one free → solutions form a line.</li>
<li>One pivot, two free → solutions form a plane.</li>
</ul>
<p>Thus, the number of free variables determines the dimension of the solution set.</p>
</section>
<section id="rank-and-free-variables" class="level4">
<h4 class="anchored" data-anchor-id="rank-and-free-variables">Rank and Free Variables</h4>
<p>The number of pivot columns equals the rank of the matrix.</p>
<p>If the coefficient matrix <span class="math inline">\(A\)</span> is <span class="math inline">\(m \times n\)</span>:</p>
<ul>
<li>Rank = number of pivots.</li>
<li>Number of free variables = <span class="math inline">\(n - \text{rank}(A)\)</span>.</li>
</ul>
<p>This is the rank–nullity connection in action:</p>
<p><span class="math display">\[
\text{number of variables} = \text{rank} + \text{nullity}.
\]</span></p>
</section>
<section id="step-by-step-example-1" class="level4">
<h4 class="anchored" data-anchor-id="step-by-step-example-1">Step-by-Step Example</h4>
<p>System:</p>
<p><span class="math display">\[
\begin{cases}  
x + 2y + z = 4 \\  
2x + 5y + z = 7  
\end{cases}
\]</span></p>
<p>Augmented matrix:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; 1 &amp; | &amp; 4 \\  
2 &amp; 5 &amp; 1 &amp; | &amp; 7  
\end{bmatrix}.
\]</span></p>
<p>Reduce:</p>
<ul>
<li><p><span class="math inline">\(R_2 \to R_2 - 2R_1\)</span> →</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; 1 &amp; | &amp; 4 \\  
0 &amp; 1 &amp; -1 &amp; | &amp; -1  
\end{bmatrix}.
\]</span></p></li>
</ul>
<p>Now:</p>
<ul>
<li>Pivot columns: 1 and 2 → variables <span class="math inline">\(x, y\)</span>.</li>
<li>Free column: 3 → variable <span class="math inline">\(z\)</span>.</li>
</ul>
<p>Solution:</p>
<p><span class="math display">\[
x = 4 - 2y - z, \quad y = -1 + z, \quad z \text{ free}.
\]</span></p>
<p>Substitute:</p>
<p><span class="math display">\[
(x, y, z) = (6 - 3z, \; -1 + z, \; z).
\]</span></p>
<p>Solutions form a line in 3D parameterized by <span class="math inline">\(z\)</span>.</p>
</section>
<section id="why-leading-ones-matter" class="level4">
<h4 class="anchored" data-anchor-id="why-leading-ones-matter">Why Leading Ones Matter</h4>
<p>In RREF, each pivot is scaled to 1, making it easy to isolate pivot variables. Without leading ones, equations may still be correct but harder to interpret.</p>
<p>For example:</p>
<p><span class="math display">\[
\begin{bmatrix}  
2 &amp; 0 &amp; | &amp; 6 \\  
0 &amp; -3 &amp; | &amp; 9  
\end{bmatrix}
\]</span></p>
<p>becomes</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 0 &amp; | &amp; 3 \\  
0 &amp; 1 &amp; | &amp; -3  
\end{bmatrix}.
\]</span></p>
<p>The solutions are immediately visible: <span class="math inline">\(x=3, y=-3\)</span>.</p>
</section>
<section id="why-it-matters-20" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-20">Why It Matters</h4>
<ol type="1">
<li>Identifying pivots shows which variables are determined and which are free.</li>
<li>The number of pivots defines rank, a central concept in linear algebra.</li>
<li>Free variables determine whether the system has a unique solution, infinitely many, or none.</li>
<li>Leading ones in RREF give immediate transparency to the solution set.</li>
</ol>
</section>
<section id="try-it-yourself-23" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-23">Try It Yourself</h4>
<ol type="1">
<li><p>Reduce</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 3 &amp; 1 &amp; | &amp; 5 \\  
2 &amp; 6 &amp; 2 &amp; | &amp; 10  
\end{bmatrix}
\]</span></p>
<p>and identify pivot and free variables.</p></li>
<li><p>For the system</p>
<p><span class="math display">\[
x + y + z = 2, \quad 2x + 3y + 5z = 7,
\]</span></p>
<p>write the RREF and express the solution with free variables.</p></li>
<li><p>Compute the rank and number of free variables of a 3×5 matrix with two pivot columns.</p></li>
<li><p>Challenge: Show that if the number of pivots equals the number of variables, the system has either no solution or a unique solution, but never infinitely many.</p></li>
</ol>
<p>Understanding pivots and free variables provides the key to classifying solution sets: unique, infinite, or none. This classification lies at the heart of solving linear systems.</p>
</section>
</section>
<section id="solving-consistent-systems" class="level3">
<h3 class="anchored" data-anchor-id="solving-consistent-systems">25. Solving Consistent Systems</h3>
<p>A system of linear equations is called consistent if it has at least one solution. Consistency is the first property to check when working with a system, because before worrying about uniqueness or parametrization, we must know whether a solution exists at all. This section explains how to recognize consistent systems, how to solve them using row-reduction, and how to describe their solutions in terms of pivots and free variables.</p>
<section id="what-consistency-means" class="level4">
<h4 class="anchored" data-anchor-id="what-consistency-means">What Consistency Means</h4>
<p>Given a system <span class="math inline">\(A\mathbf{x} = \mathbf{b}\)</span>:</p>
<ul>
<li>Consistent: At least one solution <span class="math inline">\(\mathbf{x}\)</span> satisfies the system.</li>
<li>Inconsistent: No solution exists.</li>
</ul>
<p>Consistency depends on the relationship between the vector <span class="math inline">\(\mathbf{b}\)</span> and the column space of <span class="math inline">\(A\)</span>:</p>
<p><span class="math display">\[
\mathbf{b} \in \text{Col}(A) \quad \iff \quad \text{system is consistent}.
\]</span></p>
<p>If <span class="math inline">\(\mathbf{b}\)</span> cannot be written as a linear combination of the columns of <span class="math inline">\(A\)</span>, the system has no solution.</p>
</section>
<section id="checking-consistency-with-row-reduction" class="level4">
<h4 class="anchored" data-anchor-id="checking-consistency-with-row-reduction">Checking Consistency with Row Reduction</h4>
<p>To test consistency, reduce the augmented matrix <span class="math inline">\([A | \mathbf{b}]\)</span> to row-echelon form.</p>
<ul>
<li><p>If you find a row of the form:</p>
<p><span class="math display">\[
[0 \;\; 0 \;\; \dots \;\; 0 \;|\; c], \quad c \neq 0,
\]</span></p>
<p>then the system is inconsistent (contradiction: 0 = c).</p></li>
<li><p>If no such contradiction appears, the system is consistent.</p></li>
</ul>
</section>
<section id="example-1-consistent-system-with-unique-solution" class="level4">
<h4 class="anchored" data-anchor-id="example-1-consistent-system-with-unique-solution">Example 1: Consistent System with Unique Solution</h4>
<p>System:</p>
<p><span class="math display">\[
\begin{cases}  
x + y = 2 \\  
x - y = 0  
\end{cases}
\]</span></p>
<p>Augmented matrix:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 1 &amp; | &amp; 2 \\  
1 &amp; -1 &amp; | &amp; 0  
\end{bmatrix}.
\]</span></p>
<p>Row reduce:</p>
<ul>
<li><p><span class="math inline">\(R_2 \to R_2 - R_1\)</span>:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 1 &amp; | &amp; 2 \\  
0 &amp; -2 &amp; | &amp; -2  
\end{bmatrix}.
\]</span></p></li>
<li><p><span class="math inline">\(R_2 \to -\tfrac{1}{2}R_2\)</span>:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 1 &amp; | &amp; 2 \\  
0 &amp; 1 &amp; | &amp; 1  
\end{bmatrix}.
\]</span></p></li>
<li><p><span class="math inline">\(R_1 \to R_1 - R_2\)</span>:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 0 &amp; | &amp; 1 \\  
0 &amp; 1 &amp; | &amp; 1  
\end{bmatrix}.
\]</span></p></li>
</ul>
<p>Solution: <span class="math inline">\(x = 1, \; y = 1\)</span>. Unique solution.</p>
</section>
<section id="example-2-consistent-system-with-infinitely-many-solutions" class="level4">
<h4 class="anchored" data-anchor-id="example-2-consistent-system-with-infinitely-many-solutions">Example 2: Consistent System with Infinitely Many Solutions</h4>
<p>System:</p>
<p><span class="math display">\[
\begin{cases}  
x + y + z = 3 \\  
2x + 2y + 2z = 6  
\end{cases}
\]</span></p>
<p>Augmented matrix:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 1 &amp; 1 &amp; | &amp; 3 \\  
2 &amp; 2 &amp; 2 &amp; | &amp; 6  
\end{bmatrix}.
\]</span></p>
<p>Row reduce:</p>
<ul>
<li><p><span class="math inline">\(R_2 \to R_2 - 2R_1\)</span>:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 1 &amp; 1 &amp; | &amp; 3 \\  
0 &amp; 0 &amp; 0 &amp; | &amp; 0  
\end{bmatrix}.
\]</span></p></li>
</ul>
<p>No contradiction, so consistent. Solution:</p>
<p><span class="math display">\[
x = 3 - y - z, \quad y \text{ free}, \quad z \text{ free}.
\]</span></p>
<p>The solution set is a plane in <span class="math inline">\(\mathbb{R}^3\)</span>.</p>
</section>
<section id="example-3-inconsistent-system-for-contrast" class="level4">
<h4 class="anchored" data-anchor-id="example-3-inconsistent-system-for-contrast">Example 3: Inconsistent System (for contrast)</h4>
<p>System:</p>
<p><span class="math display">\[
\begin{cases}  
x + y = 1 \\  
x + y = 2  
\end{cases}
\]</span></p>
<p>Augmented matrix:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 1 &amp; | &amp; 1 \\  
1 &amp; 1 &amp; | &amp; 2  
\end{bmatrix}.
\]</span></p>
<p>Row reduce:</p>
<ul>
<li><p><span class="math inline">\(R_2 \to R_2 - R_1\)</span>:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 1 &amp; | &amp; 1 \\  
0 &amp; 0 &amp; | &amp; 1  
\end{bmatrix}.
\]</span></p></li>
</ul>
<p>Contradiction: <span class="math inline">\(0 = 1\)</span>. Inconsistent, no solution.</p>
</section>
<section id="geometric-interpretation-of-consistency" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-of-consistency">Geometric Interpretation of Consistency</h4>
<ul>
<li><p>In 2D:</p>
<ul>
<li>Two lines intersect at a point → consistent, unique solution.</li>
<li>Two lines overlap → consistent, infinitely many solutions.</li>
<li>Two lines are parallel and distinct → inconsistent, no solution.</li>
</ul></li>
<li><p>In 3D:</p>
<ul>
<li>Three planes intersect at a point → unique solution.</li>
<li>Planes intersect along a line or coincide → infinitely many solutions.</li>
<li>Planes fail to meet (like a triangular “gap”) → no solution.</li>
</ul></li>
</ul>
</section>
<section id="pivot-structure-and-solutions" class="level4">
<h4 class="anchored" data-anchor-id="pivot-structure-and-solutions">Pivot Structure and Solutions</h4>
<ul>
<li>Unique solution: Every variable is a pivot variable (no free variables).</li>
<li>Infinitely many solutions: At least one free variable exists, but no contradiction.</li>
<li>No solution: Contradictory row appears in augmented matrix.</li>
</ul>
</section>
<section id="why-it-matters-21" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-21">Why It Matters</h4>
<ol type="1">
<li>Consistency is the first checkpoint in solving systems.</li>
<li>The classification into unique, infinite, or none underpins all of linear algebra.</li>
<li>Understanding consistency ties algebra (row operations) to geometry (intersections of lines, planes, hyperplanes).</li>
<li>These ideas scale: in data science and engineering, checking whether equations are consistent is equivalent to asking if a model fits observed data.</li>
</ol>
</section>
<section id="try-it-yourself-24" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-24">Try It Yourself</h4>
<ol type="1">
<li><p>Reduce the augmented matrix</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; 1 &amp; | &amp; 5 \\  
2 &amp; 4 &amp; 2 &amp; | &amp; 10 \\  
3 &amp; 6 &amp; 3 &amp; | &amp; 15  
\end{bmatrix}
\]</span></p>
<p>and determine if the system is consistent.</p></li>
<li><p>Classify the system as having unique, infinite, or no solutions:</p>
<p><span class="math display">\[
\begin{cases}  
x + y + z = 2 \\  
x - y + z = 0 \\  
2x + 0y + 2z = 3  
\end{cases}
\]</span></p></li>
<li><p>Explain geometrically what it means when the augmented matrix has a contradictory row.</p></li>
<li><p>Challenge: Show algebraically that a system is consistent if and only if <span class="math inline">\(\mathbf{b}\)</span> lies in the span of the columns of <span class="math inline">\(A\)</span>.</p></li>
</ol>
<p>Consistent systems mark the balance point between algebraic rules and geometric reality: they are where equations and space meet in harmony.</p>
</section>
</section>
<section id="detecting-inconsistency" class="level3">
<h3 class="anchored" data-anchor-id="detecting-inconsistency">26. Detecting Inconsistency</h3>
<p>Not every system of linear equations has a solution. Some are inconsistent, meaning the equations contradict one another and no vector <span class="math inline">\(\mathbf{x}\)</span> can satisfy them all at once. Detecting such inconsistency early is crucial: it saves wasted effort trying to solve an impossible system and reveals important geometric and algebraic properties.</p>
<section id="what-inconsistency-looks-like-algebraically" class="level4">
<h4 class="anchored" data-anchor-id="what-inconsistency-looks-like-algebraically">What Inconsistency Looks Like Algebraically</h4>
<p>Consider the system:</p>
<p><span class="math display">\[
\begin{cases}  
x + y = 1 \\  
x + y = 3  
\end{cases}
\]</span></p>
<p>Clearly, the two equations cannot both be true. In augmented matrix form:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 1 &amp; | &amp; 1 \\  
1 &amp; 1 &amp; | &amp; 3  
\end{bmatrix}.
\]</span></p>
<p>Row reduction gives:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 1 &amp; | &amp; 1 \\  
0 &amp; 0 &amp; | &amp; 2  
\end{bmatrix}.
\]</span></p>
<p>The bottom row says <span class="math inline">\(0 = 2\)</span>, a contradiction. This is the hallmark of inconsistency: a row of zeros in the coefficient part, with a nonzero constant in the augmented part.</p>
</section>
<section id="general-rule-for-detection" class="level4">
<h4 class="anchored" data-anchor-id="general-rule-for-detection">General Rule for Detection</h4>
<p>A system <span class="math inline">\(A\mathbf{x} = \mathbf{b}\)</span> is inconsistent if, after row reduction, the augmented matrix contains a row of the form:</p>
<p><span class="math display">\[
[0 \;\; 0 \;\; \dots \;\; 0 \;|\; c], \quad c \neq 0.
\]</span></p>
<p>This indicates that all variables vanish from the equation, leaving an impossible statement like <span class="math inline">\(0 = c\)</span>.</p>
</section>
<section id="example-1-parallel-lines-in-2d" class="level4">
<h4 class="anchored" data-anchor-id="example-1-parallel-lines-in-2d">Example 1: Parallel Lines in 2D</h4>
<p><span class="math display">\[
\begin{cases}  
x + y = 2 \\  
2x + 2y = 5  
\end{cases}
\]</span></p>
<p>Augmented matrix:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 1 &amp; | &amp; 2 \\  
2 &amp; 2 &amp; | &amp; 5  
\end{bmatrix}.
\]</span></p>
<p>Row reduce:</p>
<ul>
<li><span class="math inline">\(R_2 \to R_2 - 2R_1\)</span>:</li>
</ul>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 1 &amp; | &amp; 2 \\  
0 &amp; 0 &amp; | &amp; 1  
\end{bmatrix}.
\]</span></p>
<p>Contradiction: no solution. Geometrically, the two equations are parallel lines that never intersect.</p>
</section>
<section id="example-2-contradictory-planes-in-3d" class="level4">
<h4 class="anchored" data-anchor-id="example-2-contradictory-planes-in-3d">Example 2: Contradictory Planes in 3D</h4>
<p><span class="math display">\[
\begin{cases}  
x + y + z = 1 \\  
2x + 2y + 2z = 2 \\  
x + y + z = 3  
\end{cases}
\]</span></p>
<p>The first and third equations already conflict: the same plane equation is forced to equal two different constants.</p>
<p>Augmented matrix reduces to:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 1 &amp; 1 &amp; | &amp; 1 \\  
0 &amp; 0 &amp; 0 &amp; | &amp; 0 \\  
0 &amp; 0 &amp; 0 &amp; | &amp; 2  
\end{bmatrix}.
\]</span></p>
<p>Contradiction: no solution. The “planes” fail to intersect in common.</p>
</section>
<section id="geometry-of-inconsistency" class="level4">
<h4 class="anchored" data-anchor-id="geometry-of-inconsistency">Geometry of Inconsistency</h4>
<ul>
<li>In 2D: Inconsistent systems correspond to parallel lines with different intercepts.</li>
<li>In 3D: They correspond to planes that are parallel but offset, or planes arranged in a way that leaves a “gap” (no shared intersection).</li>
<li>In higher dimensions: Inconsistency means the target vector <span class="math inline">\(\mathbf{b}\)</span> lies outside the column space of <span class="math inline">\(A\)</span>.</li>
</ul>
</section>
<section id="rank-test-for-consistency" class="level4">
<h4 class="anchored" data-anchor-id="rank-test-for-consistency">Rank Test for Consistency</h4>
<p>Another way to detect inconsistency is using ranks.</p>
<ul>
<li>Let <span class="math inline">\(\text{rank}(A)\)</span> be the number of pivots in the coefficient matrix.</li>
<li>Let <span class="math inline">\(\text{rank}([A|\mathbf{b}])\)</span> be the number of pivots in the augmented matrix.</li>
</ul>
<p>Rule:</p>
<ul>
<li>If <span class="math inline">\(\text{rank}(A) = \text{rank}([A|\mathbf{b}])\)</span>, the system is consistent.</li>
<li>If <span class="math inline">\(\text{rank}(A) &lt; \text{rank}([A|\mathbf{b}])\)</span>, the system is inconsistent.</li>
</ul>
<p>This rank condition is fundamental and works in any dimension.</p>
</section>
<section id="why-it-matters-22" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-22">Why It Matters</h4>
<ol type="1">
<li>Inconsistency reveals overdetermined or contradictory data in real problems (physics, engineering, statistics).</li>
<li>The ability to detect inconsistency quickly through row reduction or rank saves computation.</li>
<li>It connects geometry (non-intersecting spaces) with algebra (contradictory rows).</li>
<li>It prepares the way for least-squares methods, where inconsistent systems are approximated instead of solved exactly.</li>
</ol>
</section>
<section id="try-it-yourself-25" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-25">Try It Yourself</h4>
<ol type="1">
<li>Reduce the augmented matrix</li>
</ol>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; -1 &amp; | &amp; 2 \\  
2 &amp; -2 &amp; | &amp; 5  
\end{bmatrix}
\]</span></p>
<p>and decide if the system is consistent.</p>
<ol start="2" type="1">
<li>Show geometrically why the system</li>
</ol>
<p><span class="math display">\[
x + y = 0, \quad x + y = 1
\]</span></p>
<p>is inconsistent.</p>
<ol start="3" type="1">
<li>Use the rank test to check consistency of</li>
</ol>
<p><span class="math display">\[
\begin{cases}  
x + y + z = 2 \\  
2x + 2y + 2z = 4 \\  
3x + 3y + 3z = 5  
\end{cases}
\]</span></p>
<ol start="4" type="1">
<li>Challenge: Explain why <span class="math inline">\(\text{rank}(A) &lt; \text{rank}([A|\mathbf{b}])\)</span> implies inconsistency, using the concept of the column space.</li>
</ol>
<p>Detecting inconsistency is not just about spotting contradictions-it connects algebra, geometry, and linear transformations, showing exactly when a system cannot possibly fit together.</p>
</section>
</section>
<section id="gaussian-elimination-by-hand" class="level3">
<h3 class="anchored" data-anchor-id="gaussian-elimination-by-hand">27. Gaussian Elimination by Hand</h3>
<p>Gaussian elimination is the systematic procedure for solving systems of linear equations by using row operations to simplify the augmented matrix. The goal is to transform the system into row-echelon form (REF) and then use back substitution to find the solutions. This method is the backbone of linear algebra computations and is the foundation of most computer algorithms for solving linear systems.</p>
<section id="the-big-idea" class="level4">
<h4 class="anchored" data-anchor-id="the-big-idea">The Big Idea</h4>
<ol type="1">
<li>Represent the system as an augmented matrix.</li>
<li>Use row operations to eliminate variables step by step, moving left to right, top to bottom.</li>
<li>Stop when the matrix is in REF.</li>
<li>Solve the triangular system by back substitution.</li>
</ol>
</section>
<section id="step-by-step-recipe" class="level4">
<h4 class="anchored" data-anchor-id="step-by-step-recipe">Step-by-Step Recipe</h4>
<p>Suppose we have <span class="math inline">\(n\)</span> equations with <span class="math inline">\(n\)</span> unknowns.</p>
<ol type="1">
<li>Choose a pivot in the first column (a nonzero entry). If needed, swap rows to bring a nonzero entry to the top.</li>
<li>Eliminate below the pivot by subtracting multiples of the pivot row from lower rows so that all entries below the pivot become zero.</li>
<li>Move to the next row and next column, pick the next pivot, and repeat elimination.</li>
<li>Continue until all pivots are in stair-step form (REF).</li>
<li>Use back substitution to solve for the unknowns starting from the bottom row.</li>
</ol>
</section>
<section id="example-1-a-22-system" class="level4">
<h4 class="anchored" data-anchor-id="example-1-a-22-system">Example 1: A 2×2 System</h4>
<p>System:</p>
<p><span class="math display">\[
\begin{cases}  
x + 2y = 5 \\  
3x + 4y = 11  
\end{cases}
\]</span></p>
<p>Augmented matrix:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; | &amp; 5 \\  
3 &amp; 4 &amp; | &amp; 11  
\end{bmatrix}.
\]</span></p>
<ol type="1">
<li><p>Pivot at (1,1) = 1.</p></li>
<li><p>Eliminate below: <span class="math inline">\(R_2 \to R_2 - 3R_1\)</span>.</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; | &amp; 5 \\  
0 &amp; -2 &amp; | &amp; -4  
\end{bmatrix}.
\]</span></p></li>
<li><p>Back substitution: From row 2: <span class="math inline">\(-2y = -4 \implies y = 2\)</span>. Substitute into row 1: <span class="math inline">\(x + 2(2) = 5 \implies x = 1\)</span>.</p></li>
</ol>
<p>Solution: <span class="math inline">\((x, y) = (1, 2)\)</span>.</p>
</section>
<section id="example-2-a-33-system" class="level4">
<h4 class="anchored" data-anchor-id="example-2-a-33-system">Example 2: A 3×3 System</h4>
<p>System:</p>
<p><span class="math display">\[
\begin{cases}  
x + y + z = 6 \\  
2x + 3y + z = 14 \\  
x - y + 2z = 2  
\end{cases}
\]</span></p>
<p>Augmented matrix:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 1 &amp; 1 &amp; | &amp; 6 \\  
2 &amp; 3 &amp; 1 &amp; | &amp; 14 \\  
1 &amp; -1 &amp; 2 &amp; | &amp; 2  
\end{bmatrix}.
\]</span></p>
<p>Step 1: Pivot at (1,1). Eliminate below:</p>
<ul>
<li><span class="math inline">\(R_2 \to R_2 - 2R_1\)</span>.</li>
<li><span class="math inline">\(R_3 \to R_3 - R_1\)</span>.</li>
</ul>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 1 &amp; 1 &amp; | &amp; 6 \\  
0 &amp; 1 &amp; -1 &amp; | &amp; 2 \\  
0 &amp; -2 &amp; 1 &amp; | &amp; -4  
\end{bmatrix}.
\]</span></p>
<p>Step 2: Pivot at (2,2). Eliminate below: <span class="math inline">\(R_3 \to R_3 + 2R_2\)</span>.</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 1 &amp; 1 &amp; | &amp; 6 \\  
0 &amp; 1 &amp; -1 &amp; | &amp; 2 \\  
0 &amp; 0 &amp; -1 &amp; | &amp; 0  
\end{bmatrix}.
\]</span></p>
<p>Step 3: Pivot at (3,3). Scale row: <span class="math inline">\(R_3 \to -R_3\)</span>.</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 1 &amp; 1 &amp; | &amp; 6 \\  
0 &amp; 1 &amp; -1 &amp; | &amp; 2 \\  
0 &amp; 0 &amp; 1 &amp; | &amp; 0  
\end{bmatrix}.
\]</span></p>
<p>Back substitution:</p>
<ul>
<li>From row 3: <span class="math inline">\(z = 0\)</span>.</li>
<li>From row 2: <span class="math inline">\(y - z = 2 \implies y = 2\)</span>.</li>
<li>From row 1: <span class="math inline">\(x + y + z = 6 \implies x = 4\)</span>.</li>
</ul>
<p>Solution: <span class="math inline">\((x, y, z) = (4, 2, 0)\)</span>.</p>
</section>
<section id="why-gaussian-elimination-always-works" class="level4">
<h4 class="anchored" data-anchor-id="why-gaussian-elimination-always-works">Why Gaussian Elimination Always Works</h4>
<ul>
<li>Each step reduces the number of variables in the lower equations.</li>
<li>Pivoting ensures stability (swap rows to avoid dividing by zero).</li>
<li>The algorithm either produces a triangular system (solvable by substitution) or reveals inconsistency (contradictory row).</li>
</ul>
</section>
<section id="geometric-interpretation-1" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-1">Geometric Interpretation</h4>
<ul>
<li><p>Elimination corresponds to progressively restricting the solution set:</p>
<ul>
<li>First equation → a plane in <span class="math inline">\(\mathbb{R}^3\)</span>.</li>
<li>Add second equation → intersection becomes a line.</li>
<li>Add third equation → intersection becomes a point (unique solution) or vanishes (inconsistent).</li>
</ul></li>
</ul>
</section>
<section id="why-it-matters-23" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-23">Why It Matters</h4>
<ol type="1">
<li>Gaussian elimination is the foundation for solving systems by hand and by computer.</li>
<li>It reveals whether a system is consistent and if solutions are unique or infinite.</li>
<li>It is the starting point for advanced methods like LU decomposition, QR factorization, and numerical solvers.</li>
<li>It shows the interplay between algebra (row operations) and geometry (intersections of subspaces).</li>
</ol>
</section>
<section id="try-it-yourself-26" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-26">Try It Yourself</h4>
<ol type="1">
<li><p>Solve the system</p>
<p><span class="math display">\[
\begin{cases}  
2x + y = 7 \\  
4x + 3y = 15  
\end{cases}
\]</span></p>
<p>using Gaussian elimination.</p></li>
<li><p>Reduce</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; -1 &amp; | &amp; 3 \\  
3 &amp; 8 &amp; 1 &amp; | &amp; 12 \\  
2 &amp; 6 &amp; 3 &amp; | &amp; 11  
\end{bmatrix}
\]</span></p>
<p>to REF and solve.</p></li>
<li><p>Practice with a system that has infinitely many solutions:</p>
<p><span class="math display">\[
x + y + z = 4, \quad 2x + 2y + 2z = 8.
\]</span></p></li>
<li><p>Challenge: Explain why Gaussian elimination always terminates in at most <span class="math inline">\(n\)</span> pivot steps for an <span class="math inline">\(n \times n\)</span> system.</p></li>
</ol>
<p>Gaussian elimination transforms the complexity of many equations into an orderly process, making the hidden structure of solutions visible step by step.</p>
</section>
</section>
<section id="back-substitution-and-solution-sets" class="level3">
<h3 class="anchored" data-anchor-id="back-substitution-and-solution-sets">28. Back Substitution and Solution Sets</h3>
<p>Once Gaussian elimination reduces a system to row-echelon form (REF), the next step is to actually solve for the unknowns. This process is called back substitution: we begin with the bottom equation (which involves the fewest variables) and work our way upward, solving step by step. Back substitution is what converts the structured triangular system into explicit solutions.</p>
<section id="the-structure-of-row-echelon-form" class="level4">
<h4 class="anchored" data-anchor-id="the-structure-of-row-echelon-form">The Structure of Row-Echelon Form</h4>
<p>A system in REF looks like this:</p>
<p><span class="math display">\[
\begin{bmatrix}  
- &amp; * &amp; * &amp; * &amp; | &amp; * \\  
0 &amp; * &amp; * &amp; * &amp; | &amp; * \\  
0 &amp; 0 &amp; * &amp; * &amp; | &amp; * \\  
0 &amp; 0 &amp; 0 &amp; * &amp; | &amp; *  
\end{bmatrix}
\]</span></p>
<ul>
<li>Each row corresponds to an equation with fewer variables than the row above.</li>
<li>The bottom equation has only one or two variables.</li>
<li>This triangular form makes it possible to solve “from the bottom up.”</li>
</ul>
</section>
<section id="step-by-step-example-unique-solution" class="level4">
<h4 class="anchored" data-anchor-id="step-by-step-example-unique-solution">Step-by-Step Example: Unique Solution</h4>
<p>System after elimination:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; -1 &amp; | &amp; 3 \\  
0 &amp; 1 &amp; 2 &amp; | &amp; 4 \\  
0 &amp; 0 &amp; 1 &amp; | &amp; 2  
\end{bmatrix}.
\]</span></p>
<p>This corresponds to:</p>
<p><span class="math display">\[
\begin{cases}  
x + 2y - z = 3 \\  
y + 2z = 4 \\  
z = 2  
\end{cases}
\]</span></p>
<ol type="1">
<li>From the last equation: <span class="math inline">\(z = 2\)</span>.</li>
<li>Substitute into the second: <span class="math inline">\(y + 2(2) = 4 \implies y = 0\)</span>.</li>
<li>Substitute into the first: <span class="math inline">\(x + 2(0) - 2 = 3 \implies x = 5\)</span>.</li>
</ol>
<p>Solution: <span class="math inline">\((x, y, z) = (5, 0, 2)\)</span>.</p>
</section>
<section id="infinite-solutions-with-free-variables" class="level4">
<h4 class="anchored" data-anchor-id="infinite-solutions-with-free-variables">Infinite Solutions with Free Variables</h4>
<p>Not all systems reduce to unique solutions. If there are free variables (non-pivot columns), back substitution expresses pivot variables in terms of free ones.</p>
<p>Example:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; 1 &amp; | &amp; 4 \\  
0 &amp; 1 &amp; -1 &amp; | &amp; 1 \\  
0 &amp; 0 &amp; 0 &amp; | &amp; 0  
\end{bmatrix}.
\]</span></p>
<p>Equations:</p>
<p><span class="math display">\[
\begin{cases}  
x + 2y + z = 4 \\  
y - z = 1  
\end{cases}
\]</span></p>
<ol type="1">
<li>From row 2: <span class="math inline">\(y = 1 + z\)</span>.</li>
<li>From row 1: <span class="math inline">\(x + 2(1 + z) + z = 4 \implies x = 2 - 3z\)</span>.</li>
</ol>
<p>Solution set:</p>
<p><span class="math display">\[
(x, y, z) = (2 - 3t, \; 1 + t, \; t), \quad t \in \mathbb{R}.
\]</span></p>
<p>Here <span class="math inline">\(z = t\)</span> is the free variable. The solutions form a line in 3D.</p>
</section>
<section id="general-solution-structure" class="level4">
<h4 class="anchored" data-anchor-id="general-solution-structure">General Solution Structure</h4>
<p>For a consistent system:</p>
<ol type="1">
<li>Unique solution → every variable is a pivot variable (no free variables).</li>
<li>Infinitely many solutions → some free variables remain. The solution set is parametrized by these variables and forms a line, plane, or higher-dimensional subspace.</li>
<li>No solution → contradiction discovered earlier, so back substitution is impossible.</li>
</ol>
</section>
<section id="geometric-meaning-2" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-2">Geometric Meaning</h4>
<ul>
<li>Unique solution → a single intersection point of lines/planes.</li>
<li>Infinite solutions → overlapping subspaces (e.g., two planes intersecting in a line).</li>
<li>Back substitution describes the exact shape of this intersection.</li>
</ul>
</section>
<section id="example-parametric-vector-form" class="level4">
<h4 class="anchored" data-anchor-id="example-parametric-vector-form">Example: Parametric Vector Form</h4>
<p>For the infinite-solution example above:</p>
<p><span class="math display">\[
(x, y, z) = (2, 1, 0) + t(-3, 1, 1).
\]</span></p>
<p>This expresses the solution set as a base point plus a direction vector, making the geometry clear.</p>
</section>
<section id="why-it-matters-24" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-24">Why It Matters</h4>
<ol type="1">
<li>Back substitution turns row-echelon form into concrete answers.</li>
<li>It distinguishes unique vs.&nbsp;infinite solutions.</li>
<li>It provides a systematic method usable by hand for small systems and forms the basis of computer algorithms for large ones.</li>
<li>It reveals the structure of solution sets-whether a point, line, plane, or higher-dimensional object.</li>
</ol>
</section>
<section id="try-it-yourself-27" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-27">Try It Yourself</h4>
<ol type="1">
<li>Solve by back substitution:</li>
</ol>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; -1 &amp; 2 &amp; | &amp; 3 \\  
0 &amp; 1 &amp; 3 &amp; | &amp; 5 \\  
0 &amp; 0 &amp; 1 &amp; | &amp; 2  
\end{bmatrix}.
\]</span></p>
<ol start="2" type="1">
<li>Reduce and solve:</li>
</ol>
<p><span class="math display">\[
x + y + z = 2, \quad 2x + 2y + 2z = 4.
\]</span></p>
<ol start="3" type="1">
<li><p>Express the solution set of the above system in parametric vector form.</p></li>
<li><p>Challenge: For a 4×4 system with two free variables, explain why the solution set forms a plane in <span class="math inline">\(\mathbb{R}^4\)</span>.</p></li>
</ol>
<p>Back substitution completes the elimination process, translating triangular structure into explicit solutions, and shows how algebra and geometry meet in the classification of solution sets.</p>
</section>
</section>
<section id="rank-and-its-first-meaning" class="level3">
<h3 class="anchored" data-anchor-id="rank-and-its-first-meaning">29. Rank and Its First Meaning</h3>
<p>The concept of rank lies at the heart of linear algebra. It connects the algebra of solving systems, the geometry of subspaces, and the structure of matrices into one unifying idea. Rank measures the amount of independent information in a matrix: how many rows or columns carry unique directions instead of being repetitions or combinations of others.</p>
<section id="definition-of-rank" class="level4">
<h4 class="anchored" data-anchor-id="definition-of-rank">Definition of Rank</h4>
<p>The rank of a matrix <span class="math inline">\(A\)</span> is the number of pivots in its row-echelon form. Equivalently, it is:</p>
<ul>
<li>The dimension of the column space (number of independent columns).</li>
<li>The dimension of the row space (number of independent rows).</li>
</ul>
<p>All these definitions agree.</p>
</section>
<section id="first-encounter-with-rank-pivot-counting" class="level4">
<h4 class="anchored" data-anchor-id="first-encounter-with-rank-pivot-counting">First Encounter with Rank: Pivot Counting</h4>
<p>When solving a system with Gaussian elimination:</p>
<ul>
<li>Every pivot corresponds to one determined variable.</li>
<li>The number of pivots = the rank.</li>
<li>The number of free variables = total variables – rank.</li>
</ul>
<p>Example:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; 1 &amp; | &amp; 4 \\  
0 &amp; 1 &amp; -1 &amp; | &amp; 2 \\  
0 &amp; 0 &amp; 0 &amp; | &amp; 0  
\end{bmatrix}.
\]</span></p>
<p>Here, there are 2 pivots. So:</p>
<ul>
<li>Rank = 2.</li>
<li>With 3 variables total, there is 1 free variable.</li>
</ul>
</section>
<section id="rank-in-terms-of-independence" class="level4">
<h4 class="anchored" data-anchor-id="rank-in-terms-of-independence">Rank in Terms of Independence</h4>
<p>A set of vectors is linearly independent if none can be expressed as a combination of the others.</p>
<ul>
<li>The rank of a matrix tells us how many independent rows or columns it has.</li>
<li>If some columns are combinations of others, they do not increase the rank.</li>
</ul>
<p>Example:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; 3 \\  
2 &amp; 4 &amp; 6 \\  
3 &amp; 6 &amp; 9  
\end{bmatrix}.
\]</span></p>
<p>Here, each row is a multiple of the first. Rank = 1, since only one independent row/column direction exists.</p>
</section>
<section id="rank-and-solutions-of-systems" class="level4">
<h4 class="anchored" data-anchor-id="rank-and-solutions-of-systems">Rank and Solutions of Systems</h4>
<p>Consider <span class="math inline">\(A\mathbf{x} = \mathbf{b}\)</span>.</p>
<ul>
<li>If <span class="math inline">\(\text{rank}(A) = \text{rank}([A|\mathbf{b}])\)</span>, the system is consistent.</li>
<li>If not, inconsistent.</li>
<li>If rank = number of variables, the system has a unique solution.</li>
<li>If rank &lt; number of variables, there are infinitely many solutions.</li>
</ul>
<p>Thus, rank classifies solution sets.</p>
</section>
<section id="rank-and-geometry" class="level4">
<h4 class="anchored" data-anchor-id="rank-and-geometry">Rank and Geometry</h4>
<p>Rank tells us the dimension of the subspace spanned by rows or columns.</p>
<ul>
<li>Rank 1: all information lies along a line.</li>
<li>Rank 2: lies in a plane.</li>
<li>Rank 3: fills 3D space.</li>
</ul>
<p>Example:</p>
<ul>
<li>In <span class="math inline">\(\mathbb{R}^3\)</span>, a matrix of rank 2 has columns spanning a plane through the origin.</li>
<li>A matrix of rank 1 has all columns on a single line.</li>
</ul>
</section>
<section id="rank-and-row-vs.-column-view" class="level4">
<h4 class="anchored" data-anchor-id="rank-and-row-vs.-column-view">Rank and Row vs.&nbsp;Column View</h4>
<p>It is a remarkable fact that the number of independent rows = number of independent columns. This is not obvious at first glance, but it is always true. So we can define rank either by rows or by columns-it makes no difference.</p>
</section>
<section id="why-it-matters-25" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-25">Why It Matters</h4>
<ol type="1">
<li>Rank is the bridge between algebra and geometry: pivots ↔︎ dimension.</li>
<li>It classifies solutions to systems of equations.</li>
<li>It measures redundancy in data (important in statistics, machine learning, signal processing).</li>
<li>It prepares the way for advanced concepts like nullity, rank–nullity theorem, and singular value decomposition.</li>
</ol>
</section>
<section id="try-it-yourself-28" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-28">Try It Yourself</h4>
<ol type="1">
<li><p>Find the rank of</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; 3 \\  
2 &amp; 4 &amp; 5 \\  
3 &amp; 6 &amp; 8  
\end{bmatrix}.
\]</span></p></li>
<li><p>Solve the system</p>
<p><span class="math display">\[
x + y + z = 2, \quad 2x + 2y + 2z = 4,
\]</span></p>
<p>and identify the rank of the coefficient matrix.</p></li>
<li><p>In <span class="math inline">\(\mathbb{R}^3\)</span>, what is the geometric meaning of a 3×3 matrix of rank 2?</p></li>
<li><p>Challenge: Prove that the row rank always equals the column rank by considering the echelon form of the matrix.</p></li>
</ol>
<p>Rank is the first truly unifying concept in linear algebra: it tells us how much independent structure a matrix contains and sets the stage for understanding spaces, dimensions, and transformations.</p>
</section>
</section>
<section id="lu-factorization" class="level3">
<h3 class="anchored" data-anchor-id="lu-factorization">30. LU Factorization</h3>
<p>Gaussian elimination not only solves systems but also reveals a deeper structure: many matrices can be factored into simpler pieces. One of the most useful is the LU factorization, where a matrix <span class="math inline">\(A\)</span> is written as the product of a lower-triangular matrix <span class="math inline">\(L\)</span> and an upper-triangular matrix <span class="math inline">\(U\)</span>. This factorization captures all the elimination steps in a compact form and allows systems to be solved efficiently.</p>
<section id="what-is-lu-factorization" class="level4">
<h4 class="anchored" data-anchor-id="what-is-lu-factorization">What is LU Factorization?</h4>
<p>If <span class="math inline">\(A\)</span> is an <span class="math inline">\(n \times n\)</span> matrix, then</p>
<p><span class="math display">\[
A = LU,
\]</span></p>
<p>where:</p>
<ul>
<li><span class="math inline">\(L\)</span> is lower-triangular (entries below diagonal may be nonzero, diagonal entries = 1).</li>
<li><span class="math inline">\(U\)</span> is upper-triangular (entries above diagonal may be nonzero).</li>
</ul>
<p>This means:</p>
<ul>
<li><span class="math inline">\(U\)</span> stores the result of elimination (the triangular system).</li>
<li><span class="math inline">\(L\)</span> records the multipliers used during elimination.</li>
</ul>
</section>
<section id="example-22-case" class="level4">
<h4 class="anchored" data-anchor-id="example-22-case">Example: 2×2 Case</h4>
<p>Take</p>
<p><span class="math display">\[
A = \begin{bmatrix}  
2 &amp; 3 \\  
4 &amp; 7  
\end{bmatrix}.
\]</span></p>
<p>Elimination: <span class="math inline">\(R_2 \to R_2 - 2R_1\)</span>.</p>
<ul>
<li><p>Multiplier = 2 (used to eliminate entry 4).</p></li>
<li><p>Resulting <span class="math inline">\(U\)</span>:</p>
<p><span class="math display">\[
U = \begin{bmatrix}  
2 &amp; 3 \\  
0 &amp; 1  
\end{bmatrix}.
\]</span></p></li>
<li><p><span class="math inline">\(L\)</span>:</p>
<p><span class="math display">\[
L = \begin{bmatrix}  
1 &amp; 0 \\  
2 &amp; 1  
\end{bmatrix}.
\]</span></p></li>
</ul>
<p>Check:</p>
<p><span class="math display">\[
LU = \begin{bmatrix}  
1 &amp; 0 \\  
2 &amp; 1  
\end{bmatrix}  
\begin{bmatrix}  
2 &amp; 3 \\  
0 &amp; 1  
\end{bmatrix}  
= \begin{bmatrix}  
2 &amp; 3 \\  
4 &amp; 7  
\end{bmatrix} = A.
\]</span></p>
</section>
<section id="example-33-case" class="level4">
<h4 class="anchored" data-anchor-id="example-33-case">Example: 3×3 Case</h4>
<p><span class="math display">\[
A = \begin{bmatrix}  
2 &amp; 1 &amp; 1 \\  
4 &amp; -6 &amp; 0 \\  
-2 &amp; 7 &amp; 2  
\end{bmatrix}.
\]</span></p>
<p>Step 1: Eliminate below pivot (row 1).</p>
<ul>
<li>Multiplier <span class="math inline">\(m_{21} = 4/2 = 2\)</span>.</li>
<li>Multiplier <span class="math inline">\(m_{31} = -2/2 = -1\)</span>.</li>
</ul>
<p>Step 2: Eliminate below pivot in column 2.</p>
<ul>
<li>After substitutions, multipliers and pivots are collected.</li>
</ul>
<p>Result:</p>
<p><span class="math display">\[
L = \begin{bmatrix}  
1 &amp; 0 &amp; 0 \\  
2 &amp; 1 &amp; 0 \\  
-1 &amp; -1 &amp; 1  
\end{bmatrix}, \quad  
U = \begin{bmatrix}  
2 &amp; 1 &amp; 1 \\  
0 &amp; -8 &amp; -2 \\  
0 &amp; 0 &amp; 1  
\end{bmatrix}.
\]</span></p>
<p>Thus <span class="math inline">\(A = LU\)</span>.</p>
</section>
<section id="solving-systems-with-lu" class="level4">
<h4 class="anchored" data-anchor-id="solving-systems-with-lu">Solving Systems with LU</h4>
<p>Suppose <span class="math inline">\(Ax = b\)</span>. If <span class="math inline">\(A = LU\)</span>:</p>
<ol type="1">
<li>Solve <span class="math inline">\(Ly = b\)</span> by forward substitution (since <span class="math inline">\(L\)</span> is lower-triangular).</li>
<li>Solve <span class="math inline">\(Ux = y\)</span> by back substitution (since <span class="math inline">\(U\)</span> is upper-triangular).</li>
</ol>
<p>This two-step process is much faster than elimination from scratch each time, especially if solving multiple systems with the same <span class="math inline">\(A\)</span> but different <span class="math inline">\(b\)</span>.</p>
</section>
<section id="pivoting-and-permutations" class="level4">
<h4 class="anchored" data-anchor-id="pivoting-and-permutations">Pivoting and Permutations</h4>
<p>Sometimes elimination requires row swaps (to avoid division by zero or instability). Then factorization is written as:</p>
<p><span class="math display">\[
PA = LU,
\]</span></p>
<p>where <span class="math inline">\(P\)</span> is a permutation matrix recording the row swaps. This is the practical form used in numerical computing.</p>
</section>
<section id="applications-of-lu-factorization" class="level4">
<h4 class="anchored" data-anchor-id="applications-of-lu-factorization">Applications of LU Factorization</h4>
<ol type="1">
<li>Efficient solving: Multiple right-hand sides <span class="math inline">\(Ax = b\)</span>. Compute <span class="math inline">\(LU\)</span> once, reuse for each <span class="math inline">\(b\)</span>.</li>
<li>Determinants: <span class="math inline">\(\det(A) = \det(L)\det(U)\)</span>. Since diagonals of <span class="math inline">\(L\)</span> are 1, this reduces to the product of the diagonal of <span class="math inline">\(U\)</span>.</li>
<li>Matrix inverse: By solving <span class="math inline">\(Ax = e_i\)</span> for each column <span class="math inline">\(e_i\)</span>, we can compute <span class="math inline">\(A^{-1}\)</span> efficiently with LU.</li>
<li>Numerical methods: LU is central in scientific computing, engineering simulations, and optimization.</li>
</ol>
</section>
<section id="geometric-meaning-3" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-3">Geometric Meaning</h4>
<p>LU decomposition separates the elimination process into:</p>
<ul>
<li><span class="math inline">\(L\)</span>: shear transformations (adding multiples of rows).</li>
<li><span class="math inline">\(U\)</span>: scaling and alignment into triangular form.</li>
</ul>
<p>Together, they represent the same linear transformation as <span class="math inline">\(A\)</span>, but decomposed into simpler building blocks.</p>
</section>
<section id="why-it-matters-26" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-26">Why It Matters</h4>
<ol type="1">
<li>LU factorization compresses elimination into a reusable format.</li>
<li>It is a cornerstone of numerical linear algebra and used in almost every solver.</li>
<li>It links computation (efficient algorithms) with theory (factorization of transformations).</li>
<li>It introduces the broader idea that matrices can be broken into simple, interpretable parts.</li>
</ol>
</section>
<section id="try-it-yourself-29" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-29">Try It Yourself</h4>
<ol type="1">
<li><p>Factor</p>
<p><span class="math display">\[
A = \begin{bmatrix}  
1 &amp; 2 \\  
3 &amp; 8  
\end{bmatrix}
\]</span></p>
<p>into <span class="math inline">\(LU\)</span>.</p></li>
<li><p>Solve</p>
<p><span class="math display">\[
\begin{bmatrix}  
2 &amp; 1 \\  
6 &amp; 3  
\end{bmatrix}  
\begin{bmatrix}  
x \\ y  
\end{bmatrix} =  
\begin{bmatrix}  
5 \\ 15  
\end{bmatrix}
\]</span></p>
<p>using LU decomposition.</p></li>
<li><p>Compute <span class="math inline">\(\det(A)\)</span> for</p>
<p><span class="math display">\[
A = \begin{bmatrix}  
2 &amp; 1 &amp; 1 \\  
4 &amp; -6 &amp; 0 \\  
-2 &amp; 7 &amp; 2  
\end{bmatrix}
\]</span></p>
<p>by using its LU factorization.</p></li>
<li><p>Challenge: Prove that if <span class="math inline">\(A\)</span> is invertible, then it has an LU factorization (possibly after row swaps).</p></li>
</ol>
<p>LU factorization organizes elimination into a powerful tool: compact, efficient, and deeply tied to both the theory and practice of linear algebra.</p>
</section>
<section id="closing-2" class="level4">
<h4 class="anchored" data-anchor-id="closing-2">Closing</h4>
<pre><code>Paths diverge or merge,
pivots mark the way forward,
truth distilled in rows.</code></pre>
</section>
</section>
</section>
<section id="chapter-4.-vector-spaces-and-subspaces" class="level2">
<h2 class="anchored" data-anchor-id="chapter-4.-vector-spaces-and-subspaces">Chapter 4. Vector spaces and subspaces</h2>
<section id="opening-2" class="level4">
<h4 class="anchored" data-anchor-id="opening-2">Opening</h4>
<pre><code>Endless skies expand,
spaces within spaces grow,
freedom takes its shape.</code></pre>
</section>
<section id="axioms-of-vector-spaces" class="level3">
<h3 class="anchored" data-anchor-id="axioms-of-vector-spaces">31. Axioms of Vector Spaces</h3>
<p>Up to now, we have worked with vectors in <span class="math inline">\(\mathbb{R}^2\)</span>, <span class="math inline">\(\mathbb{R}^3\)</span>, and higher-dimensional Euclidean spaces. But the true power of linear algebra comes from abstracting away from coordinates. A vector space is not tied to arrows in physical space-it is any collection of objects that behave like vectors, provided they satisfy certain rules. These rules are called the axioms of vector spaces.</p>
<section id="the-idea-of-a-vector-space" class="level4">
<h4 class="anchored" data-anchor-id="the-idea-of-a-vector-space">The Idea of a Vector Space</h4>
<p>A vector space is a set <span class="math inline">\(V\)</span> equipped with two operations:</p>
<ol type="1">
<li>Vector addition: Combine two vectors in <span class="math inline">\(V\)</span> to get another vector in <span class="math inline">\(V\)</span>.</li>
<li>Scalar multiplication: Multiply a vector in <span class="math inline">\(V\)</span> by a scalar (a number from a field, usually <span class="math inline">\(\mathbb{R}\)</span> or <span class="math inline">\(\mathbb{C}\)</span>).</li>
</ol>
<p>The magic is that as long as certain rules (axioms) hold, the objects in <span class="math inline">\(V\)</span> can be treated as vectors. They need not be arrows or coordinate lists-they could be polynomials, functions, matrices, or sequences.</p>
</section>
<section id="the-eight-axioms" class="level4">
<h4 class="anchored" data-anchor-id="the-eight-axioms">The Eight Axioms</h4>
<p>Let <span class="math inline">\(u, v, w \in V\)</span> (vectors) and <span class="math inline">\(a, b \in \mathbb{R}\)</span> (scalars). The axioms are:</p>
<ol type="1">
<li>Closure under addition: <span class="math inline">\(u + v \in V\)</span>.</li>
<li>Commutativity of addition: <span class="math inline">\(u + v = v + u\)</span>.</li>
<li>Associativity of addition: <span class="math inline">\((u + v) + w = u + (v + w)\)</span>.</li>
<li>Existence of additive identity: There exists a zero vector <span class="math inline">\(0 \in V\)</span> such that <span class="math inline">\(v + 0 = v\)</span>.</li>
<li>Existence of additive inverses: For every <span class="math inline">\(v\)</span>, there is <span class="math inline">\(-v\)</span> such that <span class="math inline">\(v + (-v) = 0\)</span>.</li>
<li>Closure under scalar multiplication: <span class="math inline">\(a v \in V\)</span>.</li>
<li>Distributivity of scalar multiplication over vector addition: <span class="math inline">\(a(u + v) = au + av\)</span>.</li>
<li>Distributivity of scalar multiplication over scalar addition: <span class="math inline">\((a + b)v = av + bv\)</span>.</li>
<li>Associativity of scalar multiplication: <span class="math inline">\(a(bv) = (ab)v\)</span>.</li>
<li>Existence of multiplicative identity: <span class="math inline">\(1 \cdot v = v\)</span>.</li>
</ol>
<p>(These are sometimes listed as eight, with some grouped together, but the essence is the same.)</p>
</section>
<section id="examples-of-vector-spaces" class="level4">
<h4 class="anchored" data-anchor-id="examples-of-vector-spaces">Examples of Vector Spaces</h4>
<ol type="1">
<li>Euclidean spaces: <span class="math inline">\(\mathbb{R}^n\)</span> with standard addition and scalar multiplication.</li>
<li>Polynomials: The set of all polynomials with real coefficients, <span class="math inline">\(\mathbb{R}[x]\)</span>.</li>
<li>Functions: The set of all continuous functions on <span class="math inline">\([0,1]\)</span>, with addition of functions and scalar multiplication.</li>
<li>Matrices: The set of all <span class="math inline">\(m \times n\)</span> matrices with real entries.</li>
<li>Sequences: The set of all infinite real sequences <span class="math inline">\((a_1, a_2, \dots)\)</span>.</li>
</ol>
<p>All of these satisfy the vector space axioms.</p>
</section>
<section id="non-examples" class="level4">
<h4 class="anchored" data-anchor-id="non-examples">Non-Examples</h4>
<ol type="1">
<li>The set of natural numbers <span class="math inline">\(\mathbb{N}\)</span> is not a vector space (no additive inverses).</li>
<li>The set of positive real numbers <span class="math inline">\(\mathbb{R}^+\)</span> is not a vector space (not closed under scalar multiplication with negative numbers).</li>
<li>The set of polynomials of degree exactly 2 is not a vector space (not closed under addition: <span class="math inline">\(x^2 + x^2 = 2x^2\)</span> is still degree 2, but <span class="math inline">\(x^2 - x^2 = 0\)</span>, which is degree 0, not allowed).</li>
</ol>
<p>These examples show why the axioms are essential: without them, the structure breaks.</p>
</section>
<section id="the-zero-vector" class="level4">
<h4 class="anchored" data-anchor-id="the-zero-vector">The Zero Vector</h4>
<p>Every vector space must contain a zero vector. This is not optional. It is the “do nothing” element for addition. In <span class="math inline">\(\mathbb{R}^n\)</span>, this is <span class="math inline">\((0,0,\dots,0)\)</span>. In polynomials, it is the zero polynomial. In function spaces, it is the function <span class="math inline">\(f(x) = 0\)</span>.</p>
</section>
<section id="additive-inverses" class="level4">
<h4 class="anchored" data-anchor-id="additive-inverses">Additive Inverses</h4>
<p>For every vector <span class="math inline">\(v\)</span>, we require <span class="math inline">\(-v\)</span>. This ensures that equations like <span class="math inline">\(u+v=w\)</span> can always be rearranged to <span class="math inline">\(u=w-v\)</span>. Without additive inverses, solving linear equations would not work.</p>
</section>
<section id="scalars-and-fields" class="level4">
<h4 class="anchored" data-anchor-id="scalars-and-fields">Scalars and Fields</h4>
<p>Scalars come from a field: usually the real numbers <span class="math inline">\(\mathbb{R}\)</span> or the complex numbers <span class="math inline">\(\mathbb{C}\)</span>. The choice of scalars matters:</p>
<ul>
<li>Over <span class="math inline">\(\mathbb{R}\)</span>, a polynomial space is different from over <span class="math inline">\(\mathbb{C}\)</span>.</li>
<li>Over finite fields (like integers modulo <span class="math inline">\(p\)</span>), vector spaces exist in discrete mathematics and coding theory.</li>
</ul>
</section>
<section id="geometric-interpretation-2" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-2">Geometric Interpretation</h4>
<ul>
<li>The axioms guarantee that vectors can be added and scaled in predictable ways.</li>
<li>Closure ensures the space is “self-contained.”</li>
<li>Additive inverses ensure symmetry: every direction can be reversed.</li>
<li>Distributivity ensures consistency between scaling and addition.</li>
</ul>
<p>Together, these rules make vector spaces stable and reliable mathematical objects.</p>
</section>
<section id="why-it-matters-27" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-27">Why It Matters</h4>
<ol type="1">
<li>Vector spaces unify many areas of math under a single framework.</li>
<li>They generalize <span class="math inline">\(\mathbb{R}^n\)</span> to functions, polynomials, and beyond.</li>
<li>The axioms guarantee that all the tools of linear algebra-span, basis, dimension, linear maps-apply.</li>
<li>Recognizing vector spaces in disguise is a major step in advanced math and physics.</li>
</ol>
</section>
<section id="try-it-yourself-30" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-30">Try It Yourself</h4>
<ol type="1">
<li>Verify that the set of all 2×2 matrices is a vector space under matrix addition and scalar multiplication.</li>
<li>Show that the set of polynomials of degree at most 3 is a vector space, but the set of polynomials of degree exactly 3 is not.</li>
<li>Check whether the set of all even functions <span class="math inline">\(f(-x) = f(x)\)</span> is a vector space.</li>
<li>Challenge: Consider the set of all differentiable functions <span class="math inline">\(f\)</span> on <span class="math inline">\([0,1]\)</span>. Show that this set forms a vector space under the usual operations.</li>
</ol>
<p>The axioms of vector spaces provide the foundation on which the rest of linear algebra is built. Everything that follows-subspaces, independence, basis, dimension-grows naturally from this formal framework.</p>
</section>
</section>
<section id="subspaces-column-space-and-null-space" class="level3">
<h3 class="anchored" data-anchor-id="subspaces-column-space-and-null-space">32. Subspaces, Column Space, and Null Space</h3>
<p>Once the idea of a vector space is in place, the next step is to recognize smaller vector spaces that live inside bigger ones. These are called subspaces. Subspaces are central in linear algebra because they reveal the internal structure of matrices and linear systems. Two special subspaces-the column space and the null space-play particularly important roles.</p>
<section id="what-is-a-subspace" class="level4">
<h4 class="anchored" data-anchor-id="what-is-a-subspace">What Is a Subspace?</h4>
<p>A subspace <span class="math inline">\(W\)</span> of a vector space <span class="math inline">\(V\)</span> is a subset of <span class="math inline">\(V\)</span> that is itself a vector space under the same operations. To qualify as a subspace, <span class="math inline">\(W\)</span> must satisfy:</p>
<ol type="1">
<li>The zero vector <span class="math inline">\(0\)</span> is in <span class="math inline">\(W\)</span>.</li>
<li>If <span class="math inline">\(u, v \in W\)</span>, then <span class="math inline">\(u+v \in W\)</span> (closed under addition).</li>
<li>If <span class="math inline">\(u \in W\)</span> and <span class="math inline">\(c\)</span> is a scalar, then <span class="math inline">\(cu \in W\)</span> (closed under scalar multiplication).</li>
</ol>
<p>That’s it-no further checking of all ten vector space axioms is needed, because those are inherited from <span class="math inline">\(V\)</span>.</p>
</section>
<section id="simple-examples-of-subspaces" class="level4">
<h4 class="anchored" data-anchor-id="simple-examples-of-subspaces">Simple Examples of Subspaces</h4>
<ul>
<li><p>In <span class="math inline">\(\mathbb{R}^3\)</span>:</p>
<ul>
<li>A line through the origin is a 1-dimensional subspace.</li>
<li>A plane through the origin is a 2-dimensional subspace.</li>
<li>The whole space itself is a subspace.</li>
<li>The trivial subspace <span class="math inline">\(\{0\}\)</span> contains only the zero vector.</li>
</ul></li>
<li><p>In the space of polynomials:</p>
<ul>
<li>All polynomials of degree ≤ 3 form a subspace.</li>
<li>All polynomials with zero constant term form a subspace.</li>
</ul></li>
<li><p>In function spaces:</p>
<ul>
<li>All continuous functions on <span class="math inline">\([0,1]\)</span> form a subspace of all functions on <span class="math inline">\([0,1]\)</span>.</li>
<li>All solutions to a linear differential equation form a subspace.</li>
</ul></li>
</ul>
</section>
<section id="the-column-space-of-a-matrix" class="level4">
<h4 class="anchored" data-anchor-id="the-column-space-of-a-matrix">The Column Space of a Matrix</h4>
<p>Given a matrix <span class="math inline">\(A\)</span>, the column space is the set of all linear combinations of its columns. Formally,</p>
<p><span class="math display">\[
C(A) = \{ A\mathbf{x} : \mathbf{x} \in \mathbb{R}^n \}.
\]</span></p>
<ul>
<li>The column space lives inside <span class="math inline">\(\mathbb{R}^m\)</span> if <span class="math inline">\(A\)</span> is <span class="math inline">\(m \times n\)</span>.</li>
<li>It represents all possible outputs of the linear transformation defined by <span class="math inline">\(A\)</span>.</li>
<li>Its dimension is equal to the rank of <span class="math inline">\(A\)</span>.</li>
</ul>
<p>Example:</p>
<p><span class="math display">\[
A = \begin{bmatrix}  
1 &amp; 2 \\  
2 &amp; 4 \\  
3 &amp; 6  
\end{bmatrix}.
\]</span></p>
<p>The second column is just twice the first. So the column space is all multiples of <span class="math inline">\(\begin{bmatrix}1 \\ 2 \\ 3\end{bmatrix}\)</span>, which is a line in <span class="math inline">\(\mathbb{R}^3\)</span>. Rank = 1.</p>
</section>
<section id="the-null-space-of-a-matrix" class="level4">
<h4 class="anchored" data-anchor-id="the-null-space-of-a-matrix">The Null Space of a Matrix</h4>
<p>The null space (or kernel) of a matrix <span class="math inline">\(A\)</span> is the set of all vectors <span class="math inline">\(\mathbf{x}\)</span> such that</p>
<p><span class="math display">\[
A\mathbf{x} = 0.
\]</span></p>
<ul>
<li>It lives in <span class="math inline">\(\mathbb{R}^n\)</span> if <span class="math inline">\(A\)</span> is <span class="math inline">\(m \times n\)</span>.</li>
<li>It represents the “invisible” directions that collapse to zero under the transformation.</li>
<li>Its dimension is the nullity of <span class="math inline">\(A\)</span>.</li>
</ul>
<p>Example:</p>
<p><span class="math display">\[
A = \begin{bmatrix}  
1 &amp; 2 &amp; 3 \\  
4 &amp; 5 &amp; 6  
\end{bmatrix}.
\]</span></p>
<p>Solve <span class="math inline">\(A\mathbf{x} = 0\)</span>. This yields a null space spanned by one vector, meaning it is a line through the origin in <span class="math inline">\(\mathbb{R}^3\)</span>.</p>
</section>
<section id="column-space-vs.-null-space" class="level4">
<h4 class="anchored" data-anchor-id="column-space-vs.-null-space">Column Space vs.&nbsp;Null Space</h4>
<ul>
<li>Column space: describes outputs (<span class="math inline">\(y\)</span>-values that can be reached).</li>
<li>Null space: describes hidden inputs (directions that vanish).</li>
</ul>
<p>Together, they capture the full behavior of a matrix.</p>
</section>
<section id="geometric-interpretation-3" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-3">Geometric Interpretation</h4>
<ul>
<li>In <span class="math inline">\(\mathbb{R}^3\)</span>, the column space could be a plane or a line inside 3D space.</li>
<li>The null space is orthogonal (in a precise sense) to the row space, which we’ll study later.</li>
<li>Understanding both spaces gives a complete picture of how the matrix transforms vectors.</li>
</ul>
</section>
<section id="why-it-matters-28" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-28">Why It Matters</h4>
<ol type="1">
<li>Subspaces are the natural habitat of linear algebra: almost everything happens inside them.</li>
<li>The column space explains what systems <span class="math inline">\(Ax=b\)</span> are solvable.</li>
<li>The null space explains why some systems have multiple solutions (free variables).</li>
<li>These ideas extend to advanced topics like eigenvectors, SVD, and differential equations.</li>
</ol>
</section>
<section id="try-it-yourself-31" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-31">Try It Yourself</h4>
<ol type="1">
<li><p>Show that the set <span class="math inline">\(\{(x,y,0) : x,y \in \mathbb{R}\}\)</span> is a subspace of <span class="math inline">\(\mathbb{R}^3\)</span>.</p></li>
<li><p>For</p>
<p><span class="math display">\[
A = \begin{bmatrix}  
1 &amp; 2 &amp; 3 \\  
0 &amp; 0 &amp; 0 \\  
1 &amp; 2 &amp; 3  
\end{bmatrix},
\]</span></p>
<p>find the column space and its dimension.</p></li>
<li><p>For the same <span class="math inline">\(A\)</span>, compute the null space and its dimension.</p></li>
<li><p>Challenge: Prove that the null space of <span class="math inline">\(A\)</span> is always a subspace of <span class="math inline">\(\mathbb{R}^n\)</span>.</p></li>
</ol>
<p>Subspaces-especially the column space and null space-are the first glimpse of the hidden geometry inside every matrix, showing us which directions survive and which vanish.</p>
</section>
</section>
<section id="span-and-generating-sets" class="level3">
<h3 class="anchored" data-anchor-id="span-and-generating-sets">33. Span and Generating Sets</h3>
<p>The idea of a span captures the simplest and most powerful way to build new vectors from old ones: by taking linear combinations. A span is not just a set of scattered points but a structured, complete collection of all combinations of a given set of vectors. Understanding span leads directly to the concepts of bases, dimension, and the structure of subspaces.</p>
<section id="definition-of-span" class="level4">
<h4 class="anchored" data-anchor-id="definition-of-span">Definition of Span</h4>
<p>Given vectors <span class="math inline">\(v_1, v_2, \dots, v_k \in V\)</span>, the span of these vectors is</p>
<p><span class="math display">\[
\text{span}\{v_1, v_2, \dots, v_k\} = \{a_1 v_1 + a_2 v_2 + \dots + a_k v_k : a_i \in \mathbb{R}\}.
\]</span></p>
<ul>
<li>A span is the set of all possible linear combinations of the vectors.</li>
<li>It is always a subspace.</li>
<li>The given vectors are called a generating set.</li>
</ul>
</section>
<section id="simple-examples" class="level4">
<h4 class="anchored" data-anchor-id="simple-examples">Simple Examples</h4>
<ol type="1">
<li><p>In <span class="math inline">\(\mathbb{R}^2\)</span>:</p>
<ul>
<li>Span of <span class="math inline">\((1,0)\)</span> = all multiples of the x-axis (a line).</li>
<li>Span of <span class="math inline">\((1,0)\)</span> and <span class="math inline">\((0,1)\)</span> = the entire plane <span class="math inline">\(\mathbb{R}^2\)</span>.</li>
<li>Span of <span class="math inline">\((1,0)\)</span> and <span class="math inline">\((2,0)\)</span> = still the x-axis, since the second vector is redundant.</li>
</ul></li>
<li><p>In <span class="math inline">\(\mathbb{R}^3\)</span>:</p>
<ul>
<li>Span of a single vector = a line.</li>
<li>Span of two independent vectors = a plane through the origin.</li>
<li>Span of three independent vectors = the whole space <span class="math inline">\(\mathbb{R}^3\)</span>.</li>
</ul></li>
</ol>
</section>
<section id="span-as-coverage" class="level4">
<h4 class="anchored" data-anchor-id="span-as-coverage">Span as Coverage</h4>
<ul>
<li>If you think of vectors as “directions,” the span is everything you can reach by walking in those directions, with any step lengths (scalars) allowed.</li>
<li>If you only have one direction, you can walk back and forth on a line.</li>
<li>With two independent directions, you can sweep out a plane.</li>
<li>With three independent directions in 3D, you can move anywhere.</li>
</ul>
</section>
<section id="generating-sets" class="level4">
<h4 class="anchored" data-anchor-id="generating-sets">Generating Sets</h4>
<p>A set of vectors is a generating set (or spanning set) for a subspace if their span equals that subspace.</p>
<ul>
<li>Example: <span class="math inline">\(\{(1,0), (0,1)\}\)</span> generates <span class="math inline">\(\mathbb{R}^2\)</span>.</li>
<li>Example: <span class="math inline">\(\{(1,0,0), (0,1,0), (0,0,1)\}\)</span> generates <span class="math inline">\(\mathbb{R}^3\)</span>.</li>
<li>Example: The columns of a matrix generate its column space.</li>
</ul>
<p>Different generating sets can span the same space. Some may be redundant, others minimal. Later, the concept of a basis refines this idea.</p>
</section>
<section id="redundancy-in-spanning-sets" class="level4">
<h4 class="anchored" data-anchor-id="redundancy-in-spanning-sets">Redundancy in Spanning Sets</h4>
<ul>
<li>If one vector is a linear combination of others, it does not enlarge the span.</li>
<li>Example: In <span class="math inline">\(\mathbb{R}^2\)</span>, <span class="math inline">\(\{(1,0), (0,1), (1,1)\}\)</span> spans the same space as <span class="math inline">\(\{(1,0), (0,1)\}\)</span>.</li>
<li>Eliminating redundancy leads to a more efficient generating set.</li>
</ul>
</section>
<section id="span-and-linear-systems" class="level4">
<h4 class="anchored" data-anchor-id="span-and-linear-systems">Span and Linear Systems</h4>
<p>Consider the system <span class="math inline">\(Ax=b\)</span>.</p>
<ul>
<li>The question “Is there a solution?” is equivalent to “Is <span class="math inline">\(b\)</span> in the span of the columns of <span class="math inline">\(A\)</span>?”</li>
<li>Thus, span provides the geometric language for solvability.</li>
</ul>
</section>
<section id="why-it-matters-29" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-29">Why It Matters</h4>
<ol type="1">
<li>Span is the foundation for defining subspaces generated by vectors.</li>
<li>It connects directly to solvability of linear equations.</li>
<li>It introduces the notion of redundancy, preparing for bases and independence.</li>
<li>It generalizes naturally to function spaces and abstract vector spaces.</li>
</ol>
</section>
<section id="try-it-yourself-32" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-32">Try It Yourself</h4>
<ol type="1">
<li>Find the span of <span class="math inline">\(\{(1,2), (2,4)\}\)</span> in <span class="math inline">\(\mathbb{R}^2\)</span>.</li>
<li>Show that the vectors <span class="math inline">\((1,0,1), (0,1,1), (1,1,2)\)</span> span only a plane in <span class="math inline">\(\mathbb{R}^3\)</span>.</li>
<li>Decide whether <span class="math inline">\((1,2,3)\)</span> is in the span of <span class="math inline">\((1,0,1)\)</span> and <span class="math inline">\((0,1,2)\)</span>.</li>
<li>Challenge: Prove that the set of all polynomials <span class="math inline">\(\{1, x, x^2, \dots\}\)</span> spans the space of all polynomials.</li>
</ol>
<p>The concept of span transforms our perspective: instead of focusing on single vectors, we see the entire landscape of possibilities they generate.</p>
</section>
</section>
<section id="linear-independence-and-dependence" class="level3">
<h3 class="anchored" data-anchor-id="linear-independence-and-dependence">34. Linear Independence and Dependence</h3>
<p>Having introduced span and generating sets, the natural question arises: <em>when are the vectors in a spanning set truly necessary, and when are some redundant?</em> This leads to the idea of linear independence. It is the precise way to distinguish between essential vectors (those that add new directions) and dependent vectors (those that can be expressed in terms of others).</p>
<section id="definition-of-linear-independence" class="level4">
<h4 class="anchored" data-anchor-id="definition-of-linear-independence">Definition of Linear Independence</h4>
<p>A set of vectors <span class="math inline">\(\{v_1, v_2, \dots, v_k\}\)</span> is linearly independent if the only solution to</p>
<p><span class="math display">\[
a_1 v_1 + a_2 v_2 + \dots + a_k v_k = 0
\]</span></p>
<p>is</p>
<p><span class="math display">\[
a_1 = a_2 = \dots = a_k = 0.
\]</span></p>
<p>If there exists a nontrivial solution (some <span class="math inline">\(a_i \neq 0\)</span>), then the vectors are linearly dependent.</p>
</section>
<section id="intuition-1" class="level4">
<h4 class="anchored" data-anchor-id="intuition-1">Intuition</h4>
<ul>
<li>Independent vectors point in genuinely different directions.</li>
<li>Dependent vectors overlap: at least one can be built from the others.</li>
<li>In terms of span: removing a dependent vector does not shrink the span, because it adds no new direction.</li>
</ul>
</section>
<section id="simple-examples-in-mathbbr2" class="level4">
<h4 class="anchored" data-anchor-id="simple-examples-in-mathbbr2">Simple Examples in <span class="math inline">\(\mathbb{R}^2\)</span></h4>
<ol type="1">
<li><p><span class="math inline">\((1,0)\)</span> and <span class="math inline">\((0,1)\)</span> are independent.</p>
<ul>
<li>Equation <span class="math inline">\(a(1,0) + b(0,1) = (0,0)\)</span> forces <span class="math inline">\(a = b = 0\)</span>.</li>
</ul></li>
<li><p><span class="math inline">\((1,0)\)</span> and <span class="math inline">\((2,0)\)</span> are dependent.</p>
<ul>
<li>Equation <span class="math inline">\(2(1,0) - (2,0) = (0,0)\)</span> shows dependence.</li>
</ul></li>
<li><p>Any set of 3 vectors in <span class="math inline">\(\mathbb{R}^2\)</span> is dependent, since the dimension of the space is 2.</p></li>
</ol>
</section>
<section id="examples-in-mathbbr3" class="level4">
<h4 class="anchored" data-anchor-id="examples-in-mathbbr3">Examples in <span class="math inline">\(\mathbb{R}^3\)</span></h4>
<ol type="1">
<li><span class="math inline">\((1,0,0), (0,1,0), (0,0,1)\)</span> are independent.</li>
<li><span class="math inline">\((1,2,3), (2,4,6)\)</span> are dependent, since the second is just 2× the first.</li>
<li><span class="math inline">\((1,0,1), (0,1,1), (1,1,2)\)</span> are dependent: the third is the sum of the first two.</li>
</ol>
</section>
<section id="detecting-independence-with-matrices" class="level4">
<h4 class="anchored" data-anchor-id="detecting-independence-with-matrices">Detecting Independence with Matrices</h4>
<p>Put the vectors as columns in a matrix. Perform row reduction:</p>
<ul>
<li>If every column has a pivot → the set is independent.</li>
<li>If some column is free → the set is dependent.</li>
</ul>
<p>Example:</p>
<p><span class="math display">\[
\begin{bmatrix}  
1 &amp; 2 &amp; 3 \\  
0 &amp; 1 &amp; 4 \\  
0 &amp; 0 &amp; 0  
\end{bmatrix}.
\]</span></p>
<p>Here the third column has no pivot → the 3rd vector is dependent on the first two.</p>
</section>
<section id="relationship-with-dimension" class="level4">
<h4 class="anchored" data-anchor-id="relationship-with-dimension">Relationship with Dimension</h4>
<ul>
<li>In <span class="math inline">\(\mathbb{R}^n\)</span>, at most <span class="math inline">\(n\)</span> independent vectors exist.</li>
<li>If you have more than <span class="math inline">\(n\)</span>, dependence is guaranteed.</li>
<li>A basis of a vector space is simply a maximal independent set that spans the space.</li>
</ul>
</section>
<section id="geometric-interpretation-4" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-4">Geometric Interpretation</h4>
<ul>
<li>Independent vectors = different directions.</li>
<li>Dependent vectors = one vector lies in the span of others.</li>
<li>In 2D: two independent vectors span the plane.</li>
<li>In 3D: three independent vectors span the space.</li>
</ul>
</section>
<section id="why-it-matters-30" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-30">Why It Matters</h4>
<ol type="1">
<li>Independence ensures a generating set is minimal and efficient.</li>
<li>It determines whether a system of vectors is a basis.</li>
<li>It connects directly to rank: rank = number of independent columns (or rows).</li>
<li>It is crucial in geometry, data compression, and machine learning-where redundancy must be identified and removed.</li>
</ol>
</section>
<section id="try-it-yourself-33" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-33">Try It Yourself</h4>
<ol type="1">
<li>Test whether <span class="math inline">\((1,2)\)</span> and <span class="math inline">\((2,4)\)</span> are independent.</li>
<li>Are the vectors <span class="math inline">\((1,0,0), (0,1,0), (1,1,0)\)</span> independent in <span class="math inline">\(\mathbb{R}^3\)</span>?</li>
<li>Place the vectors <span class="math inline">\((1,0,1), (0,1,1), (1,1,2)\)</span> into a matrix and row-reduce to check independence.</li>
<li>Challenge: Prove that any set of <span class="math inline">\(n+1\)</span> vectors in <span class="math inline">\(\mathbb{R}^n\)</span> is linearly dependent.</li>
</ol>
<p>Linear independence is the tool that separates essential directions from redundant ones. It is the key to defining bases, counting dimensions, and understanding the structure of all vector spaces.</p>
</section>
</section>
<section id="basis-and-coordinates" class="level3">
<h3 class="anchored" data-anchor-id="basis-and-coordinates">35. Basis and Coordinates</h3>
<p>The concepts of span and linear independence come together in the powerful idea of a basis. A basis gives us the minimal set of building blocks needed to generate an entire vector space, with no redundancy. Once a basis is chosen, every vector in the space can be described uniquely by a list of numbers called its coordinates.</p>
<section id="what-is-a-basis" class="level4">
<h4 class="anchored" data-anchor-id="what-is-a-basis">What Is a Basis?</h4>
<p>A basis of a vector space <span class="math inline">\(V\)</span> is a set of vectors <span class="math inline">\(\{v_1, v_2, \dots, v_k\}\)</span> that satisfies two properties:</p>
<ol type="1">
<li>Spanning property: <span class="math inline">\(\text{span}\{v_1, \dots, v_k\} = V\)</span>.</li>
<li>Independence property: The vectors are linearly independent.</li>
</ol>
<p>In short: a basis is a spanning set with no redundancy.</p>
</section>
<section id="example-standard-bases" class="level4">
<h4 class="anchored" data-anchor-id="example-standard-bases">Example: Standard Bases</h4>
<ol type="1">
<li>In <span class="math inline">\(\mathbb{R}^2\)</span>, the standard basis is <span class="math inline">\(\{(1,0), (0,1)\}\)</span>.</li>
<li>In <span class="math inline">\(\mathbb{R}^3\)</span>, the standard basis is <span class="math inline">\(\{(1,0,0), (0,1,0), (0,0,1)\}\)</span>.</li>
<li>In <span class="math inline">\(\mathbb{R}^n\)</span>, the standard basis is the collection of unit vectors, each with a 1 in one position and 0 elsewhere.</li>
</ol>
<p>These are called standard because they are the default way of describing coordinates.</p>
</section>
<section id="uniqueness-of-coordinates" class="level4">
<h4 class="anchored" data-anchor-id="uniqueness-of-coordinates">Uniqueness of Coordinates</h4>
<p>One of the most important facts about bases is that they provide unique representations of vectors.</p>
<ul>
<li><p>Given a basis <span class="math inline">\(\{v_1, \dots, v_k\}\)</span>, any vector <span class="math inline">\(x \in V\)</span> can be written uniquely as:</p>
<p><span class="math display">\[
x = a_1 v_1 + a_2 v_2 + \dots + a_k v_k.
\]</span></p></li>
<li><p>The coefficients <span class="math inline">\((a_1, a_2, \dots, a_k)\)</span> are the coordinates of <span class="math inline">\(x\)</span> relative to that basis.</p></li>
</ul>
<p>This uniqueness distinguishes bases from arbitrary spanning sets, where redundancy allows multiple representations.</p>
</section>
<section id="example-in-mathbbr2" class="level4">
<h4 class="anchored" data-anchor-id="example-in-mathbbr2">Example in <span class="math inline">\(\mathbb{R}^2\)</span></h4>
<p>Let basis = <span class="math inline">\(\{(1,0), (0,1)\}\)</span>.</p>
<ul>
<li>Vector <span class="math inline">\((3,5) = 3(1,0) + 5(0,1)\)</span>.</li>
<li>Coordinates relative to this basis: <span class="math inline">\((3,5)\)</span>.</li>
</ul>
<p>If we switch to a different basis, the coordinates change even though the vector itself does not.</p>
</section>
<section id="example-with-non-standard-basis" class="level4">
<h4 class="anchored" data-anchor-id="example-with-non-standard-basis">Example with Non-Standard Basis</h4>
<p>Basis = <span class="math inline">\(\{(1,1), (1,-1)\}\)</span> in <span class="math inline">\(\mathbb{R}^2\)</span>. Find coordinates of <span class="math inline">\(x = (2,0)\)</span>.</p>
<p>Solve <span class="math inline">\(a(1,1) + b(1,-1) = (2,0)\)</span>. This gives system:</p>
<p><span class="math display">\[
a + b = 2, \quad a - b = 0.
\]</span></p>
<p>So <span class="math inline">\(a=1, b=1\)</span>. Coordinates relative to this basis: <span class="math inline">\((1,1)\)</span>.</p>
<p>Notice: coordinates depend on basis choice.</p>
</section>
<section id="basis-of-function-spaces" class="level4">
<h4 class="anchored" data-anchor-id="basis-of-function-spaces">Basis of Function Spaces</h4>
<ol type="1">
<li><p>For polynomials of degree ≤ 2: basis = <span class="math inline">\(\{1, x, x^2\}\)</span>.</p>
<ul>
<li>Example: <span class="math inline">\(2 + 3x + 5x^2\)</span> has coordinates <span class="math inline">\((2,3,5)\)</span>.</li>
</ul></li>
<li><p>For continuous functions on <span class="math inline">\([0,1]\)</span>, one possible basis is the infinite set <span class="math inline">\(\{1, x, x^2, \dots\}\)</span>.</p></li>
</ol>
<p>This shows bases are not restricted to geometric vectors.</p>
</section>
<section id="dimension" class="level4">
<h4 class="anchored" data-anchor-id="dimension">Dimension</h4>
<p>The number of vectors in a basis is the dimension of the vector space.</p>
<ul>
<li><span class="math inline">\(\mathbb{R}^2\)</span> has dimension 2.</li>
<li><span class="math inline">\(\mathbb{R}^3\)</span> has dimension 3.</li>
<li>The space of polynomials of degree ≤ 3 has dimension 4.</li>
</ul>
<p>Dimension tells us how many independent directions exist in the space.</p>
</section>
<section id="change-of-basis" class="level4">
<h4 class="anchored" data-anchor-id="change-of-basis">Change of Basis</h4>
<ul>
<li>Switching from one basis to another is like translating between languages.</li>
<li>The same vector looks different depending on which “dictionary” (basis) you use.</li>
<li>Change-of-basis matrices allow systematic translation between coordinate systems.</li>
</ul>
</section>
<section id="geometric-interpretation-5" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-5">Geometric Interpretation</h4>
<ul>
<li>A basis is like setting up coordinate axes in a space.</li>
<li>In 2D, two independent vectors define a grid.</li>
<li>In 3D, three independent vectors define a full coordinate system.</li>
<li>Different bases = different grids overlaying the same space.</li>
</ul>
</section>
<section id="why-it-matters-31" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-31">Why It Matters</h4>
<ol type="1">
<li>Bases provide the simplest possible description of a vector space.</li>
<li>They allow us to assign unique coordinates to vectors.</li>
<li>They connect the abstract structure of a space with concrete numerical representations.</li>
<li>The concept underlies almost all of linear algebra: dimension, transformations, eigenvectors, and more.</li>
</ol>
</section>
<section id="try-it-yourself-34" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-34">Try It Yourself</h4>
<ol type="1">
<li>Show that <span class="math inline">\(\{(1,2), (3,4)\}\)</span> is a basis of <span class="math inline">\(\mathbb{R}^2\)</span>.</li>
<li>Express <span class="math inline">\((4,5)\)</span> in terms of basis <span class="math inline">\(\{(1,1), (1,-1)\}\)</span>.</li>
<li>Prove that no basis of <span class="math inline">\(\mathbb{R}^3\)</span> can have more than 3 vectors.</li>
<li>Challenge: Show that the set <span class="math inline">\(\{1, \cos x, \sin x\}\)</span> is a basis for the space of all linear combinations of <span class="math inline">\(1, \cos x, \sin x\)</span>.</li>
</ol>
<p>A basis is the minimal, elegant foundation of a vector space, turning the infinite into the manageable by providing a finite set of independent building blocks.</p>
</section>
</section>
<section id="dimension-1" class="level3">
<h3 class="anchored" data-anchor-id="dimension-1">36. Dimension</h3>
<p>Dimension is one of the most profound and unifying ideas in linear algebra. It gives a single number that captures the “size” or “capacity” of a vector space: how many independent directions it has. Unlike length, width, or height in everyday geometry, dimension in linear algebra applies to spaces of any kind-geometric, algebraic, or even function spaces.</p>
<section id="definition" class="level4">
<h4 class="anchored" data-anchor-id="definition">Definition</h4>
<p>The dimension of a vector space <span class="math inline">\(V\)</span> is the number of vectors in any basis of <span class="math inline">\(V\)</span>.</p>
<ul>
<li><p>Since all bases of a vector space have the same number of elements, dimension is well-defined.</p></li>
<li><p>If <span class="math inline">\(\dim V = n\)</span>, then:</p>
<ul>
<li>Every set of more than <span class="math inline">\(n\)</span> vectors in <span class="math inline">\(V\)</span> is dependent.</li>
<li>Every set of exactly <span class="math inline">\(n\)</span> independent vectors forms a basis.</li>
</ul></li>
</ul>
</section>
<section id="examples-in-familiar-spaces" class="level4">
<h4 class="anchored" data-anchor-id="examples-in-familiar-spaces">Examples in Familiar Spaces</h4>
<ol type="1">
<li><p><span class="math inline">\(\dim(\mathbb{R}^2) = 2\)</span>.</p>
<ul>
<li>Basis: <span class="math inline">\((1,0), (0,1)\)</span>.</li>
<li>Two directions cover the whole plane.</li>
</ul></li>
<li><p><span class="math inline">\(\dim(\mathbb{R}^3) = 3\)</span>.</p>
<ul>
<li>Basis: <span class="math inline">\((1,0,0), (0,1,0), (0,0,1)\)</span>.</li>
<li>Three independent directions span 3D space.</li>
</ul></li>
<li><p>The set of all polynomials of degree ≤ 2 has dimension 3.</p>
<ul>
<li>Basis: <span class="math inline">\(\{1, x, x^2\}\)</span>.</li>
</ul></li>
<li><p>The space of all <span class="math inline">\(m \times n\)</span> matrices has dimension <span class="math inline">\(mn\)</span>.</p>
<ul>
<li>Each entry is independent, and the standard basis consists of matrices with a single 1 and the rest 0.</li>
</ul></li>
</ol>
</section>
<section id="finite-vs.-infinite-dimensions" class="level4">
<h4 class="anchored" data-anchor-id="finite-vs.-infinite-dimensions">Finite vs.&nbsp;Infinite Dimensions</h4>
<ul>
<li><p>Finite-dimensional spaces: <span class="math inline">\(\mathbb{R}^n\)</span>, polynomials of degree ≤ <span class="math inline">\(k\)</span>.</p></li>
<li><p>Infinite-dimensional spaces:</p>
<ul>
<li>The space of all polynomials (no degree limit).</li>
<li>The space of all continuous functions.</li>
<li>These cannot be spanned by a finite set of vectors.</li>
</ul></li>
</ul>
</section>
<section id="dimension-and-subspaces" class="level4">
<h4 class="anchored" data-anchor-id="dimension-and-subspaces">Dimension and Subspaces</h4>
<ul>
<li>Any subspace of <span class="math inline">\(\mathbb{R}^n\)</span> has dimension ≤ <span class="math inline">\(n\)</span>.</li>
<li>A line through the origin in <span class="math inline">\(\mathbb{R}^3\)</span>: dimension 1.</li>
<li>A plane through the origin in <span class="math inline">\(\mathbb{R}^3\)</span>: dimension 2.</li>
<li>The whole space: dimension 3.</li>
<li>The trivial subspace <span class="math inline">\(\{0\}\)</span>: dimension 0.</li>
</ul>
</section>
<section id="dimension-and-systems-of-equations" class="level4">
<h4 class="anchored" data-anchor-id="dimension-and-systems-of-equations">Dimension and Systems of Equations</h4>
<p>When solving <span class="math inline">\(A\mathbf{x} = \mathbf{b}\)</span>:</p>
<ul>
<li><p>The dimension of the column space = rank = number of independent directions in the outputs.</p></li>
<li><p>The dimension of the null space = number of free variables.</p></li>
<li><p>By the rank–nullity theorem:</p>
<p><span class="math display">\[
\dim(\text{column space}) + \dim(\text{null space}) = \text{number of variables}.
\]</span></p></li>
</ul>
</section>
<section id="geometric-meaning-4" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-4">Geometric Meaning</h4>
<ul>
<li>Dimension counts the minimum number of coordinates needed to describe a vector.</li>
<li>In <span class="math inline">\(\mathbb{R}^2\)</span>, you need 2 numbers.</li>
<li>In <span class="math inline">\(\mathbb{R}^3\)</span>, you need 3 numbers.</li>
<li>In the polynomial space of degree ≤ 3, you need 4 coefficients.</li>
</ul>
<p>Thus, dimension = length of coordinate list.</p>
</section>
<section id="checking-dimension-in-practice" class="level4">
<h4 class="anchored" data-anchor-id="checking-dimension-in-practice">Checking Dimension in Practice</h4>
<ol type="1">
<li>Place candidate vectors as columns of a matrix.</li>
<li>Row reduce to echelon form.</li>
<li>Count pivots. That number = dimension of the span of those vectors.</li>
</ol>
</section>
<section id="why-it-matters-32" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-32">Why It Matters</h4>
<ol type="1">
<li>Dimension is the most fundamental measure of a vector space.</li>
<li>It tells us how “large” or “complex” the space is.</li>
<li>It sets absolute limits: in <span class="math inline">\(\mathbb{R}^n\)</span>, no more than <span class="math inline">\(n\)</span> independent vectors exist.</li>
<li>It underlies coordinate systems, bases, and transformations.</li>
<li>It bridges geometry (lines, planes, volumes) with algebra (solutions, equations, matrices).</li>
</ol>
</section>
<section id="try-it-yourself-35" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-35">Try It Yourself</h4>
<ol type="1">
<li>What is the dimension of the span of <span class="math inline">\((1,2,3)\)</span>, <span class="math inline">\((2,4,6)\)</span>, <span class="math inline">\((0,0,0)\)</span>?</li>
<li>Find the dimension of the subspace of <span class="math inline">\(\mathbb{R}^3\)</span> defined by <span class="math inline">\(x+y+z=0\)</span>.</li>
<li>Prove that the set of all <span class="math inline">\(2 \times 2\)</span> symmetric matrices has dimension 3.</li>
<li>Challenge: Show that the space of polynomials of degree ≤ <span class="math inline">\(k\)</span> has dimension <span class="math inline">\(k+1\)</span>.</li>
</ol>
<p>Dimension is the measuring stick of linear algebra: it tells us how many independent pieces of information are needed to describe the whole space.</p>
</section>
</section>
<section id="ranknullity-theorem" class="level3">
<h3 class="anchored" data-anchor-id="ranknullity-theorem">37. Rank–Nullity Theorem</h3>
<p>The rank–nullity theorem is one of the central results of linear algebra. It gives a precise balance between two fundamental aspects of a matrix: the dimension of its column space (rank) and the dimension of its null space (nullity). It shows that no matter how complicated a matrix looks, the distribution of information between its “visible” outputs and its “hidden” null directions always obeys a strict law.</p>
<section id="statement-of-the-theorem" class="level4">
<h4 class="anchored" data-anchor-id="statement-of-the-theorem">Statement of the Theorem</h4>
<p>Let <span class="math inline">\(A\)</span> be an <span class="math inline">\(m \times n\)</span> matrix (mapping <span class="math inline">\(\mathbb{R}^n \to \mathbb{R}^m\)</span>):</p>
<p><span class="math display">\[
\text{rank}(A) + \text{nullity}(A) = n
\]</span></p>
<p>where:</p>
<ul>
<li>rank(A) = dimension of the column space of <span class="math inline">\(A\)</span>.</li>
<li>nullity(A) = dimension of the null space of <span class="math inline">\(A\)</span>.</li>
<li><span class="math inline">\(n\)</span> = number of columns of <span class="math inline">\(A\)</span>, i.e., the number of variables.</li>
</ul>
</section>
<section id="intuition-2" class="level4">
<h4 class="anchored" data-anchor-id="intuition-2">Intuition</h4>
<p>Think of a matrix as a machine that transforms input vectors into outputs:</p>
<ul>
<li>Rank measures how many independent output directions survive.</li>
<li>Nullity measures how many input directions get “lost” (mapped to zero).</li>
<li>The theorem says: total inputs = useful directions (rank) + wasted directions (nullity).</li>
</ul>
<p>This ensures nothing disappears mysteriously-every input direction is accounted for.</p>
</section>
<section id="example-1-full-rank" class="level4">
<h4 class="anchored" data-anchor-id="example-1-full-rank">Example 1: Full Rank</h4>
<p><span class="math display">\[
A = \begin{bmatrix}  
1 &amp; 0 \\  
0 &amp; 1 \\  
\end{bmatrix}.
\]</span></p>
<ul>
<li>Rank = 2 (two independent columns).</li>
<li>Null space = <span class="math inline">\(\{0\}\)</span>, so nullity = 0.</li>
<li>Rank + nullity = 2 = number of variables.</li>
</ul>
</section>
<section id="example-2-dependent-columns" class="level4">
<h4 class="anchored" data-anchor-id="example-2-dependent-columns">Example 2: Dependent Columns</h4>
<p><span class="math display">\[
A = \begin{bmatrix}  
1 &amp; 2 \\  
2 &amp; 4 \\  
3 &amp; 6 \\  
\end{bmatrix}.
\]</span></p>
<ul>
<li>Second column is a multiple of the first. Rank = 1.</li>
<li>Null space contains all vectors <span class="math inline">\((x,y)\)</span> with <span class="math inline">\(y = -2x\)</span>. Nullity = 1.</li>
<li>Rank + nullity = 1 + 1 = 2 = number of variables.</li>
</ul>
</section>
<section id="example-3-larger-system" class="level4">
<h4 class="anchored" data-anchor-id="example-3-larger-system">Example 3: Larger System</h4>
<p><span class="math display">\[
A = \begin{bmatrix}  
1 &amp; 0 &amp; 1 \\  
0 &amp; 1 &amp; 1  
\end{bmatrix}.
\]</span></p>
<ul>
<li>Columns: <span class="math inline">\((1,0), (0,1), (1,1)\)</span>.</li>
<li>Only two independent columns → Rank = 2.</li>
<li>Null space: solve <span class="math inline">\(x + z = 0, y + z = 0 \Rightarrow (x,y,z) = (-t,-t,t)\)</span>. Nullity = 1.</li>
<li>Rank + nullity = 2 + 1 = 3 = number of variables.</li>
</ul>
</section>
<section id="proof-sketch-conceptual" class="level4">
<h4 class="anchored" data-anchor-id="proof-sketch-conceptual">Proof Sketch (Conceptual)</h4>
<ol type="1">
<li><p>Row reduce <span class="math inline">\(A\)</span> to echelon form.</p></li>
<li><p>Pivots correspond to independent columns → count = rank.</p></li>
<li><p>Free variables correspond to null space directions → count = nullity.</p></li>
<li><p>Each column is either a pivot column or corresponds to a free variable, so:</p>
<p><span class="math display">\[
\text{rank} + \text{nullity} = \text{number of columns}.
\]</span></p></li>
</ol>
</section>
<section id="geometric-meaning-5" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-5">Geometric Meaning</h4>
<ul>
<li>In <span class="math inline">\(\mathbb{R}^3\)</span>, if a transformation collapses all vectors onto a plane (rank = 2), then one direction disappears entirely (nullity = 1).</li>
<li>In <span class="math inline">\(\mathbb{R}^4\)</span>, if a matrix has rank 2, then its null space has dimension 2, meaning half the input directions vanish.</li>
</ul>
<p>The theorem guarantees the geometry of “surviving” and “vanishing” directions always adds up consistently.</p>
</section>
<section id="applications-1" class="level4">
<h4 class="anchored" data-anchor-id="applications-1">Applications</h4>
<ol type="1">
<li><p>Solving systems <span class="math inline">\(Ax = b\)</span>:</p>
<ul>
<li>Rank determines consistency and structure of solutions.</li>
<li>Nullity tells how many free parameters exist in the solution.</li>
</ul></li>
<li><p>Data compression: Rank identifies independent features; nullity shows redundancy.</p></li>
<li><p>Computer graphics: Rank–nullity explains how 3D coordinates collapse into 2D images: one dimension of depth is lost.</p></li>
<li><p>Machine learning: Rank signals how much real information a dataset contains; nullity indicates degrees of freedom that add nothing new.</p></li>
</ol>
</section>
<section id="why-it-matters-33" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-33">Why It Matters</h4>
<ol type="1">
<li>The rank–nullity theorem connects the abstract ideas of rank and nullity into a single, elegant formula.</li>
<li>It ensures conservation of dimension: no information magically appears or disappears.</li>
<li>It is essential in understanding solutions of systems, dimensions of subspaces, and the structure of linear transformations.</li>
<li>It prepares the ground for deeper results in algebra, topology, and differential equations.</li>
</ol>
</section>
<section id="try-it-yourself-36" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-36">Try It Yourself</h4>
<ol type="1">
<li><p>Verify rank–nullity for</p>
<p><span class="math display">\[
A = \begin{bmatrix}  
1 &amp; 2 &amp; 3 \\  
4 &amp; 5 &amp; 6  
\end{bmatrix}.
\]</span></p></li>
<li><p>For a <span class="math inline">\(4 \times 5\)</span> matrix of rank 3, what is its nullity?</p></li>
<li><p>In <span class="math inline">\(\mathbb{R}^3\)</span>, suppose a matrix maps all of space onto a line. What are its rank and nullity?</p></li>
<li><p>Challenge: Prove rigorously that the row space and null space are orthogonal complements, and use this to derive rank–nullity again.</p></li>
</ol>
<p>The rank–nullity theorem is the law of balance in linear algebra: every input dimension is accounted for, either as a surviving direction (rank) or as one that vanishes (nullity).</p>
</section>
</section>
<section id="coordinates-relative-to-a-basis" class="level3">
<h3 class="anchored" data-anchor-id="coordinates-relative-to-a-basis">38. Coordinates Relative to a Basis</h3>
<p>Once a basis for a vector space is chosen, every vector in that space can be described uniquely in terms of the basis. These descriptions are called coordinates. Coordinates transform abstract vectors into concrete lists of numbers, making computation possible. Changing the basis changes the coordinates, but the underlying vector remains the same.</p>
<section id="the-core-idea" class="level4">
<h4 class="anchored" data-anchor-id="the-core-idea">The Core Idea</h4>
<p>Given a vector space <span class="math inline">\(V\)</span> and a basis <span class="math inline">\(B = \{v_1, v_2, \dots, v_n\}\)</span>, every vector <span class="math inline">\(x \in V\)</span> can be written uniquely as:</p>
<p><span class="math display">\[
x = a_1 v_1 + a_2 v_2 + \dots + a_n v_n.
\]</span></p>
<p>The coefficients <span class="math inline">\((a_1, a_2, \dots, a_n)\)</span> are the coordinates of <span class="math inline">\(x\)</span> with respect to the basis <span class="math inline">\(B\)</span>.</p>
<p>This representation is unique because basis vectors are independent.</p>
</section>
<section id="example-in-mathbbr2-1" class="level4">
<h4 class="anchored" data-anchor-id="example-in-mathbbr2-1">Example in <span class="math inline">\(\mathbb{R}^2\)</span></h4>
<ol type="1">
<li><p>Standard basis: <span class="math inline">\(B = \{(1,0), (0,1)\}\)</span>.</p>
<ul>
<li>Vector <span class="math inline">\(x = (3,5)\)</span>.</li>
<li>Coordinates relative to <span class="math inline">\(B\)</span>: <span class="math inline">\((3,5)\)</span>.</li>
</ul></li>
<li><p>Non-standard basis: <span class="math inline">\(B = \{(1,1), (1,-1)\}\)</span>.</p>
<ul>
<li><p>Write <span class="math inline">\(x = (3,5)\)</span> as <span class="math inline">\(a(1,1) + b(1,-1)\)</span>.</p></li>
<li><p>Solve:</p>
<p><span class="math display">\[
a+b = 3, \quad a-b = 5.
\]</span></p>
<p>Adding: <span class="math inline">\(2a = 8 \implies a = 4\)</span>. Subtracting: <span class="math inline">\(2b = -2 \implies b = -1\)</span>.</p></li>
<li><p>Coordinates relative to this basis: <span class="math inline">\((4, -1)\)</span>.</p></li>
</ul></li>
</ol>
<p>The same vector looks different depending on the chosen basis.</p>
</section>
<section id="example-in-mathbbr3" class="level4">
<h4 class="anchored" data-anchor-id="example-in-mathbbr3">Example in <span class="math inline">\(\mathbb{R}^3\)</span></h4>
<p>Let <span class="math inline">\(B = \{(1,0,0), (1,1,0), (1,1,1)\}\)</span>. Find coordinates of <span class="math inline">\(x = (2,3,4)\)</span>.</p>
<p>Solve <span class="math inline">\(a(1,0,0) + b(1,1,0) + c(1,1,1) = (2,3,4)\)</span>. This gives system:</p>
<p><span class="math display">\[
a+b+c = 2, \quad b+c = 3, \quad c = 4.
\]</span></p>
<p>From <span class="math inline">\(c=4\)</span>, we get <span class="math inline">\(b+c=3 \implies b=-1\)</span>. Then <span class="math inline">\(a+b+c=2 \implies a-1+4=2 \implies a=-1\)</span>. Coordinates: <span class="math inline">\((-1, -1, 4)\)</span>.</p>
</section>
<section id="matrix-formulation" class="level4">
<h4 class="anchored" data-anchor-id="matrix-formulation">Matrix Formulation</h4>
<p>If <span class="math inline">\(B = \{v_1, \dots, v_n\}\)</span>, form the basis matrix</p>
<p><span class="math display">\[
P = [v_1 \ v_2 \ \dots \ v_n].
\]</span></p>
<p>Then for a vector <span class="math inline">\(x\)</span>, its coordinate vector <span class="math inline">\([x]_B\)</span> satisfies</p>
<p><span class="math display">\[
P [x]_B = x.
\]</span></p>
<p>Thus,</p>
<p><span class="math display">\[
[x]_B = P^{-1}x.
\]</span></p>
<p>This shows coordinate transformation is simply matrix multiplication.</p>
</section>
<section id="changing-coordinates" class="level4">
<h4 class="anchored" data-anchor-id="changing-coordinates">Changing Coordinates</h4>
<p>Suppose a vector has coordinates <span class="math inline">\([x]_B\)</span> relative to basis <span class="math inline">\(B\)</span>. If we switch to another basis <span class="math inline">\(C\)</span>, we use a change-of-basis matrix to convert coordinates:</p>
<p><span class="math display">\[
[x]_C = (P_C^{-1} P_B) [x]_B.
\]</span></p>
<p>This process is fundamental in computer graphics, robotics, and data transformations.</p>
</section>
<section id="geometric-meaning-6" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-6">Geometric Meaning</h4>
<ul>
<li>A basis defines a coordinate system: axes in the space.</li>
<li>Coordinates are the “addresses” of vectors relative to those axes.</li>
<li>Changing basis is like rotating or stretching the grid: the address changes, but the point does not.</li>
</ul>
</section>
<section id="why-it-matters-34" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-34">Why It Matters</h4>
<ol type="1">
<li>Coordinates make abstract vectors computable.</li>
<li>They allow us to represent functions, polynomials, and geometric objects numerically.</li>
<li>Changing basis simplifies problems-e.g., diagonalization makes matrices easy to analyze.</li>
<li>They connect the abstract (spaces, bases) with the concrete (numbers, matrices).</li>
</ol>
</section>
<section id="try-it-yourself-37" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-37">Try It Yourself</h4>
<ol type="1">
<li>Express <span class="math inline">\(x=(4,2)\)</span> relative to basis <span class="math inline">\(\{(1,1),(1,-1)\}\)</span>.</li>
<li>Find coordinates of <span class="math inline">\(x=(2,1,3)\)</span> relative to basis <span class="math inline">\(\{(1,0,1),(0,1,1),(1,1,0)\}\)</span>.</li>
<li>If basis <span class="math inline">\(B\)</span> is the standard basis and basis <span class="math inline">\(C=\{(1,1),(1,-1)\}\)</span>, compute the change-of-basis matrix from <span class="math inline">\(B\)</span> to <span class="math inline">\(C\)</span>.</li>
<li>Challenge: Show that if <span class="math inline">\(P\)</span> is invertible, its columns form a basis, and explain why this guarantees uniqueness of coordinates.</li>
</ol>
<p>Coordinates relative to a basis are the bridge between geometry and algebra: they turn abstract spaces into numerical systems where computation, reasoning, and transformation become systematic and precise.</p>
</section>
</section>
<section id="change-of-basis-matrices" class="level3">
<h3 class="anchored" data-anchor-id="change-of-basis-matrices">39. Change-of-Basis Matrices</h3>
<p>Every vector space allows multiple choices of basis, and each basis provides a different way of describing the same vectors. The process of moving from one basis to another is called a change of basis. To perform this change systematically, we use a change-of-basis matrix. This matrix acts as a translator between coordinate systems: it converts the coordinates of a vector relative to one basis into coordinates relative to another.</p>
<section id="why-change-bases" class="level4">
<h4 class="anchored" data-anchor-id="why-change-bases">Why Change Bases?</h4>
<ol type="1">
<li>Simplicity of computation: Some problems are easier in certain bases. For example, diagonalizing a matrix allows us to raise it to powers more easily.</li>
<li>Geometry: Different bases can represent rotated or scaled coordinate systems.</li>
<li>Applications: In physics, computer graphics, robotics, and data science, changing bases is equivalent to switching perspectives or reference frames.</li>
</ol>
</section>
<section id="the-basic-setup" class="level4">
<h4 class="anchored" data-anchor-id="the-basic-setup">The Basic Setup</h4>
<p>Let <span class="math inline">\(V\)</span> be a vector space with two bases:</p>
<ul>
<li><span class="math inline">\(B = \{b_1, b_2, \dots, b_n\}\)</span></li>
<li><span class="math inline">\(C = \{c_1, c_2, \dots, c_n\}\)</span></li>
</ul>
<p>Suppose a vector <span class="math inline">\(x \in V\)</span> has coordinates <span class="math inline">\([x]_B\)</span> relative to <span class="math inline">\(B\)</span>, and <span class="math inline">\([x]_C\)</span> relative to <span class="math inline">\(C\)</span>.</p>
<p>We want a matrix <span class="math inline">\(P_{B \to C}\)</span> such that:</p>
<p><span class="math display">\[
[x]_C = P_{B \to C} [x]_B.
\]</span></p>
<p>This matrix <span class="math inline">\(P_{B \to C}\)</span> is the change-of-basis matrix from <span class="math inline">\(B\)</span> to <span class="math inline">\(C\)</span>.</p>
</section>
<section id="constructing-the-change-of-basis-matrix" class="level4">
<h4 class="anchored" data-anchor-id="constructing-the-change-of-basis-matrix">Constructing the Change-of-Basis Matrix</h4>
<ol type="1">
<li>Write each vector in the basis <span class="math inline">\(B\)</span> in terms of the basis <span class="math inline">\(C\)</span>.</li>
<li>Place these coordinate vectors as the columns of a matrix.</li>
<li>The resulting matrix converts coordinates from <span class="math inline">\(B\)</span> to <span class="math inline">\(C\)</span>.</li>
</ol>
<p>In matrix form:</p>
<p><span class="math display">\[
P_{B \to C} = \big[ [b_1]_C \ [b_2]_C \ \dots \ [b_n]_C \big].
\]</span></p>
</section>
<section id="example-in-mathbbr2-2" class="level4">
<h4 class="anchored" data-anchor-id="example-in-mathbbr2-2">Example in <span class="math inline">\(\mathbb{R}^2\)</span></h4>
<p>Let</p>
<ul>
<li><span class="math inline">\(B = \{(1,0), (0,1)\}\)</span> (standard basis).</li>
<li><span class="math inline">\(C = \{(1,1), (1,-1)\}\)</span>.</li>
</ul>
<p>To build <span class="math inline">\(P_{B \to C}\)</span>:</p>
<ul>
<li>Express each vector of <span class="math inline">\(B\)</span> in terms of <span class="math inline">\(C\)</span>.</li>
</ul>
<p>Solve:</p>
<p><span class="math display">\[
(1,0) = a(1,1) + b(1,-1).
\]</span></p>
<p>This gives system:</p>
<p><span class="math display">\[
a+b=1, \quad a-b=0.
\]</span></p>
<p>Solution: <span class="math inline">\(a=\tfrac{1}{2}, b=\tfrac{1}{2}\)</span>. So <span class="math inline">\((1,0) = \tfrac{1}{2}(1,1) + \tfrac{1}{2}(1,-1)\)</span>.</p>
<p>Next:</p>
<p><span class="math display">\[
(0,1) = a(1,1) + b(1,-1).
\]</span></p>
<p>System:</p>
<p><span class="math display">\[
a+b=0, \quad a-b=1.
\]</span></p>
<p>Solution: <span class="math inline">\(a=\tfrac{1}{2}, b=-\tfrac{1}{2}\)</span>.</p>
<p>Thus:</p>
<p><span class="math display">\[
P_{B \to C} = \begin{bmatrix}  
\tfrac{1}{2} &amp; \tfrac{1}{2} \\  
\tfrac{1}{2} &amp; -\tfrac{1}{2}  
\end{bmatrix}.
\]</span></p>
<p>So for any vector <span class="math inline">\(x\)</span>,</p>
<p><span class="math display">\[
[x]_C = P_{B \to C}[x]_B.
\]</span></p>
</section>
<section id="inverse-change-of-basis" class="level4">
<h4 class="anchored" data-anchor-id="inverse-change-of-basis">Inverse Change of Basis</h4>
<p>If <span class="math inline">\(P_{B \to C}\)</span> is the change-of-basis matrix from <span class="math inline">\(B\)</span> to <span class="math inline">\(C\)</span>, then its inverse is the change-of-basis matrix in the opposite direction:</p>
<p><span class="math display">\[
P_{C \to B} = (P_{B \to C})^{-1}.
\]</span></p>
<p>This makes sense: translating back and forth between languages should undo itself.</p>
</section>
<section id="general-formula-with-basis-matrices" class="level4">
<h4 class="anchored" data-anchor-id="general-formula-with-basis-matrices">General Formula with Basis Matrices</h4>
<p>Let</p>
<p><span class="math display">\[
P_B = [b_1 \ b_2 \ \dots \ b_n], \quad P_C = [c_1 \ c_2 \ \dots \ c_n],
\]</span></p>
<p>the matrices whose columns are basis vectors written in standard coordinates.</p>
<p>Then the change-of-basis matrix from <span class="math inline">\(B\)</span> to <span class="math inline">\(C\)</span> is:</p>
<p><span class="math display">\[
P_{B \to C} = P_C^{-1} P_B.
\]</span></p>
<p>This formula is extremely useful because it reduces the problem to matrix multiplication.</p>
</section>
<section id="geometric-interpretation-6" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-6">Geometric Interpretation</h4>
<ul>
<li>Changing basis is like rotating or stretching the grid lines of a coordinate system.</li>
<li>The vector itself (the point in space) does not move. What changes is its description in terms of the new grid.</li>
<li>The change-of-basis matrix is the tool that translates between these descriptions.</li>
</ul>
</section>
<section id="applications-2" class="level4">
<h4 class="anchored" data-anchor-id="applications-2">Applications</h4>
<ol type="1">
<li>Diagonalization: Expressing a matrix in a basis of its eigenvectors makes it diagonal, simplifying analysis.</li>
<li>Computer graphics: Changing camera viewpoints requires change-of-basis matrices.</li>
<li>Robotics: Coordinate transformations connect robot arms, joints, and workspace frames.</li>
<li>Data science: PCA finds a new basis (principal components) where data is easier to analyze.</li>
</ol>
</section>
<section id="why-it-matters-35" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-35">Why It Matters</h4>
<ol type="1">
<li>Provides a universal method to translate coordinates between bases.</li>
<li>Makes abstract transformations concrete and computable.</li>
<li>Forms the backbone of diagonalization, Jordan form, and the spectral theorem.</li>
<li>Connects algebraic manipulations with geometry and real-world reference frames.</li>
</ol>
</section>
<section id="try-it-yourself-38" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-38">Try It Yourself</h4>
<ol type="1">
<li>Compute the change-of-basis matrix from the standard basis to <span class="math inline">\(\{(2,1),(1,1)\}\)</span> in <span class="math inline">\(\mathbb{R}^2\)</span>.</li>
<li>Find the change-of-basis matrix from basis <span class="math inline">\(\{(1,0,0),(0,1,0),(0,0,1)\}\)</span> to <span class="math inline">\(\{(1,1,0),(0,1,1),(1,0,1)\}\)</span> in <span class="math inline">\(\mathbb{R}^3\)</span>.</li>
<li>Show that applying <span class="math inline">\(P_{B \to C}\)</span> then <span class="math inline">\(P_{C \to B}\)</span> returns the original coordinates.</li>
<li>Challenge: Derive the formula <span class="math inline">\(P_{B \to C} = P_C^{-1} P_B\)</span> starting from the definition of coordinates.</li>
</ol>
<p>Change-of-basis matrices give us the precise mechanism for switching perspectives. They ensure that although bases change, vectors remain invariant, and computations remain consistent.</p>
</section>
</section>
<section id="affine-subspaces" class="level3">
<h3 class="anchored" data-anchor-id="affine-subspaces">40. Affine Subspaces</h3>
<p>So far, vector spaces and subspaces have always passed through the origin. But in many real-world situations, we deal with shifted versions of these spaces: planes not passing through the origin, lines offset from the zero vector, or solution sets to linear equations with nonzero constants. These structures are called affine subspaces. They extend the idea of subspaces by allowing “translation away from the origin.”</p>
<section id="definition-1" class="level4">
<h4 class="anchored" data-anchor-id="definition-1">Definition</h4>
<p>An affine subspace of a vector space <span class="math inline">\(V\)</span> is a set of the form</p>
<p><span class="math display">\[
x_0 + W = \{x_0 + w : w \in W\},
\]</span></p>
<p>where:</p>
<ul>
<li><span class="math inline">\(x_0 \in V\)</span> is a fixed vector (the “base point” or “anchor”),</li>
<li><span class="math inline">\(W \subseteq V\)</span> is a linear subspace.</li>
</ul>
<p>Thus, an affine subspace is simply a subspace shifted by a vector.</p>
</section>
<section id="examples-in-mathbbr2" class="level4">
<h4 class="anchored" data-anchor-id="examples-in-mathbbr2">Examples in <span class="math inline">\(\mathbb{R}^2\)</span></h4>
<ol type="1">
<li>A line through the origin: <span class="math inline">\(\text{span}\{(1,2)\}\)</span>. This is a subspace.</li>
<li>A line not through the origin: <span class="math inline">\((3,1) + \text{span}\{(1,2)\}\)</span>. This is an affine subspace.</li>
<li>The entire plane: <span class="math inline">\(\mathbb{R}^2\)</span>, which is both a subspace and an affine subspace.</li>
</ol>
</section>
<section id="examples-in-mathbbr3-1" class="level4">
<h4 class="anchored" data-anchor-id="examples-in-mathbbr3-1">Examples in <span class="math inline">\(\mathbb{R}^3\)</span></h4>
<ol type="1">
<li>Plane through the origin: <span class="math inline">\(\text{span}\{(1,0,0),(0,1,0)\}\)</span>.</li>
<li>Plane not through the origin: <span class="math inline">\((2,3,4) + \text{span}\{(1,0,0),(0,1,0)\}\)</span>.</li>
<li>Line parallel to the z-axis but passing through <span class="math inline">\((1,1,5)\)</span>: <span class="math inline">\((1,1,5) + \text{span}\{(0,0,1)\}\)</span>.</li>
</ol>
</section>
<section id="relation-to-linear-systems" class="level4">
<h4 class="anchored" data-anchor-id="relation-to-linear-systems">Relation to Linear Systems</h4>
<p>Affine subspaces naturally arise as solution sets of linear equations.</p>
<ol type="1">
<li><p>Homogeneous system: <span class="math inline">\(Ax = 0\)</span>.</p>
<ul>
<li>Solution set is a subspace (the null space).</li>
</ul></li>
<li><p>Non-homogeneous system: <span class="math inline">\(Ax = b\)</span> with <span class="math inline">\(b \neq 0\)</span>.</p>
<ul>
<li><p>Solution set is affine.</p></li>
<li><p>If <span class="math inline">\(x_p\)</span> is one particular solution, then the general solution is:</p>
<p><span class="math display">\[
x = x_p + N(A),
\]</span></p>
<p>where <span class="math inline">\(N(A)\)</span> is the null space.</p></li>
</ul></li>
</ol>
<p>Thus, the geometry of solving equations leads naturally to affine subspaces.</p>
</section>
<section id="affine-dimension" class="level4">
<h4 class="anchored" data-anchor-id="affine-dimension">Affine Dimension</h4>
<p>The dimension of an affine subspace is defined as the dimension of its direction subspace <span class="math inline">\(W\)</span>.</p>
<ul>
<li>A point: affine subspace of dimension 0.</li>
<li>A line: dimension 1.</li>
<li>A plane: dimension 2.</li>
<li>Higher analogs continue in <span class="math inline">\(\mathbb{R}^n\)</span>.</li>
</ul>
</section>
<section id="difference-between-subspaces-and-affine-subspaces" class="level4">
<h4 class="anchored" data-anchor-id="difference-between-subspaces-and-affine-subspaces">Difference Between Subspaces and Affine Subspaces</h4>
<ul>
<li>Subspaces always contain the origin.</li>
<li>Affine subspaces may or may not pass through the origin.</li>
<li>Every subspace is an affine subspace (with base point <span class="math inline">\(x_0 = 0\)</span>).</li>
</ul>
</section>
<section id="geometric-intuition-2" class="level4">
<h4 class="anchored" data-anchor-id="geometric-intuition-2">Geometric Intuition</h4>
<p>Think of affine subspaces as “flat sheets” floating in space:</p>
<ul>
<li>A line through the origin is a rope tied at the center.</li>
<li>A line parallel to it but offset is the same rope moved to the side.</li>
<li>Affine subspaces preserve shape and direction, but not position.</li>
</ul>
</section>
<section id="applications-3" class="level4">
<h4 class="anchored" data-anchor-id="applications-3">Applications</h4>
<ol type="1">
<li>Linear equations: General solutions are affine subspaces.</li>
<li>Optimization: Feasible regions in linear programming are affine subspaces (intersected with inequalities).</li>
<li>Computer graphics: Affine transformations map affine subspaces to affine subspaces, preserving straightness and parallelism.</li>
<li>Machine learning: Affine decision boundaries (like hyperplanes) separate data into classes.</li>
</ol>
</section>
<section id="why-it-matters-36" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-36">Why It Matters</h4>
<ol type="1">
<li>Affine subspaces generalize subspaces, making linear algebra more flexible.</li>
<li>They allow us to describe solution sets that don’t include the origin.</li>
<li>They provide the geometric foundation for affine geometry, computer graphics, and optimization.</li>
<li>They serve as the bridge from pure linear algebra to applied modeling.</li>
</ol>
</section>
<section id="try-it-yourself-39" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-39">Try It Yourself</h4>
<ol type="1">
<li><p>Show that the set of solutions to</p>
<p><span class="math display">\[
x+y+z=1
\]</span></p>
<p>is an affine subspace of <span class="math inline">\(\mathbb{R}^3\)</span>. Identify its dimension.</p></li>
<li><p>Find the general solution to</p>
<p><span class="math display">\[
x+2y=3
\]</span></p>
<p>and describe it as an affine subspace.</p></li>
<li><p>Prove that the intersection of two affine subspaces is either empty or another affine subspace.</p></li>
<li><p>Challenge: Show that every affine subspace can be written uniquely as <span class="math inline">\(x_0 + W\)</span> with <span class="math inline">\(W\)</span> a subspace.</p></li>
</ol>
<p>Affine subspaces are the natural setting for most real-world linear problems: they combine the strict structure of subspaces with the freedom of translation, capturing both direction and position.</p>
</section>
<section id="closing-3" class="level4">
<h4 class="anchored" data-anchor-id="closing-3">Closing</h4>
<pre><code>Each basis a song,
dimension counts melodies,
the space breathes its form.</code></pre>
</section>
</section>
</section>
<section id="chapter-5.-linear-transformation-and-structure" class="level2">
<h2 class="anchored" data-anchor-id="chapter-5.-linear-transformation-and-structure">Chapter 5. Linear Transformation and Structure</h2>
<section id="opening-3" class="level4">
<h4 class="anchored" data-anchor-id="opening-3">Opening</h4>
<pre><code>Maps preserve the line,
reflections ripple outward,
motion kept in frame.</code></pre>
</section>
<section id="linear-transformations" class="level3">
<h3 class="anchored" data-anchor-id="linear-transformations">41. Linear Transformations</h3>
<p>A linear transformation is the heart of linear algebra. It is the rule that connects two vector spaces in a way that respects their linear structure: addition and scalar multiplication. Instead of thinking of vectors as static objects, linear transformations let us study how vectors move, stretch, rotate, project, or reflect. They give linear algebra its dynamic power and are the bridge between abstract theory and concrete applications.</p>
<section id="definition-2" class="level4">
<h4 class="anchored" data-anchor-id="definition-2">Definition</h4>
<p>A function <span class="math inline">\(T: V \to W\)</span> between vector spaces is called a linear transformation if for all <span class="math inline">\(u, v \in V\)</span> and scalars <span class="math inline">\(a, b \in \mathbb{R}\)</span> (or another field),</p>
<p><span class="math display">\[
T(au + bv) = aT(u) + bT(v).
\]</span></p>
<p>This single condition encodes two rules:</p>
<ol type="1">
<li>Additivity: <span class="math inline">\(T(u+v) = T(u) + T(v)\)</span>.</li>
<li>Homogeneity: <span class="math inline">\(T(av) = aT(v)\)</span>.</li>
</ol>
<p>If both are satisfied, the transformation is linear.</p>
</section>
<section id="examples-of-linear-transformations" class="level4">
<h4 class="anchored" data-anchor-id="examples-of-linear-transformations">Examples of Linear Transformations</h4>
<ol type="1">
<li><p>Scaling: <span class="math inline">\(T(x) = 3x\)</span> in <span class="math inline">\(\mathbb{R}\)</span>. Every number is stretched threefold.</p></li>
<li><p>Rotation in the plane: <span class="math inline">\(T(x,y) = (x\cos\theta - y\sin\theta, \, x\sin\theta + y\cos\theta)\)</span>.</p></li>
<li><p>Projection: Projecting <span class="math inline">\((x,y,z)\)</span> onto the <span class="math inline">\(xy\)</span>-plane: <span class="math inline">\(T(x,y,z) = (x,y,0)\)</span>.</p></li>
<li><p>Differentiation: On the space of polynomials, <span class="math inline">\(T(p(x)) = p'(x)\)</span>.</p></li>
<li><p>Integration: On continuous functions, <span class="math inline">\(T(f)(x) = \int_0^x f(t) \, dt\)</span>.</p></li>
</ol>
<p>All these are linear because they preserve addition and scaling.</p>
</section>
<section id="non-examples-1" class="level4">
<h4 class="anchored" data-anchor-id="non-examples-1">Non-Examples</h4>
<ul>
<li><span class="math inline">\(T(x) = x^2\)</span> is not linear, because <span class="math inline">\((x+y)^2 \neq x^2 + y^2\)</span>.</li>
<li><span class="math inline">\(T(x,y) = (x+1, y)\)</span> is not linear, because it fails homogeneity: scaling doesn’t preserve the “+1.”</li>
</ul>
<p>Nonlinear rules break the structure of vector spaces.</p>
</section>
<section id="matrix-representation" class="level4">
<h4 class="anchored" data-anchor-id="matrix-representation">Matrix Representation</h4>
<p>Every linear transformation from <span class="math inline">\(\mathbb{R}^n\)</span> to <span class="math inline">\(\mathbb{R}^m\)</span> can be represented by a matrix.</p>
<p>If <span class="math inline">\(T: \mathbb{R}^n \to \mathbb{R}^m\)</span>, then there exists an <span class="math inline">\(m \times n\)</span> matrix <span class="math inline">\(A\)</span> such that:</p>
<p><span class="math display">\[
T(x) = Ax.
\]</span></p>
<p>The columns of <span class="math inline">\(A\)</span> are simply <span class="math inline">\(T(e_1), T(e_2), \dots, T(e_n)\)</span>, where <span class="math inline">\(e_i\)</span> are the standard basis vectors.</p>
<p>Example: Let <span class="math inline">\(T(x,y) = (2x+y, x-y)\)</span>.</p>
<ul>
<li><span class="math inline">\(T(e_1) = T(1,0) = (2,1)\)</span>.</li>
<li><span class="math inline">\(T(e_2) = T(0,1) = (1,-1)\)</span>. So</li>
</ul>
<p><span class="math display">\[
A = \begin{bmatrix} 2 &amp; 1 \\ 1 &amp; -1 \end{bmatrix}.
\]</span></p>
<p>Then <span class="math inline">\(T(x,y) = A \begin{bmatrix} x \\ y \end{bmatrix}\)</span>.</p>
</section>
<section id="properties-of-linear-transformations" class="level4">
<h4 class="anchored" data-anchor-id="properties-of-linear-transformations">Properties of Linear Transformations</h4>
<ol type="1">
<li>The image of the zero vector is always zero: <span class="math inline">\(T(0) = 0\)</span>.</li>
<li>The image of a line through the origin is again a line (or collapsed to a point).</li>
<li>Composition of linear transformations is linear.</li>
<li>Every linear transformation preserves the structure of subspaces.</li>
</ol>
</section>
<section id="kernel-and-image-preview" class="level4">
<h4 class="anchored" data-anchor-id="kernel-and-image-preview">Kernel and Image (Preview)</h4>
<p>For <span class="math inline">\(T: V \to W\)</span>:</p>
<ul>
<li>The kernel (or null space) is all vectors mapped to zero: <span class="math inline">\(\ker T = \{v \in V : T(v) = 0\}\)</span>.</li>
<li>The image (or range) is all outputs that can be achieved: <span class="math inline">\(\text{im}(T) = \{T(v) : v \in V\}\)</span>. The rank–nullity theorem applies here:</li>
</ul>
<p><span class="math display">\[
\dim(\ker T) + \dim(\text{im}(T)) = \dim(V).
\]</span></p>
</section>
<section id="geometric-interpretation-7" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-7">Geometric Interpretation</h4>
<p>Linear transformations reshape space:</p>
<ul>
<li>Scaling stretches space uniformly in one direction.</li>
<li>Rotation spins space while preserving lengths.</li>
<li>Projection flattens space onto lower dimensions.</li>
<li>Reflection flips space across a line or plane.</li>
</ul>
<p>The key feature: straight lines remain straight, and the origin stays fixed.</p>
</section>
<section id="applications-4" class="level4">
<h4 class="anchored" data-anchor-id="applications-4">Applications</h4>
<ol type="1">
<li>Computer graphics: Scaling, rotating, projecting 3D objects onto 2D screens.</li>
<li>Robotics: Transformations between joint coordinates and workspace positions.</li>
<li>Data science: Linear mappings represent dimensionality reduction and feature extraction.</li>
<li>Differential equations: Solutions often involve linear operators acting on function spaces.</li>
<li>Machine learning: Weight matrices in neural networks are stacked linear transformations, interspersed with nonlinearities.</li>
</ol>
</section>
<section id="why-it-matters-37" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-37">Why It Matters</h4>
<ol type="1">
<li>Linear transformations generalize matrices to any vector space.</li>
<li>They unify geometry, algebra, and applications under one concept.</li>
<li>They provide the natural framework for studying eigenvalues, eigenvectors, and decompositions.</li>
<li>They model countless real-world processes: physical, computational, and abstract.</li>
</ol>
</section>
<section id="try-it-yourself-40" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-40">Try It Yourself</h4>
<ol type="1">
<li>Prove that <span class="math inline">\(T(x,y,z) = (x+2y, z, x-y+z)\)</span> is linear.</li>
<li>Find the matrix representation of the transformation that reflects vectors in <span class="math inline">\(\mathbb{R}^2\)</span> across the line <span class="math inline">\(y=x\)</span>.</li>
<li>Show why <span class="math inline">\(T(x,y) = (x^2,y)\)</span> is not linear.</li>
<li>Challenge: For the differentiation operator <span class="math inline">\(D: P_3 \to P_2\)</span> on polynomials of degree ≤ 3, find its matrix relative to the basis <span class="math inline">\(\{1,x,x^2,x^3\}\)</span> in the domain and <span class="math inline">\(\{1,x,x^2\}\)</span> in the codomain.</li>
</ol>
<p>Linear transformations are the language of linear algebra. They capture the essence of symmetry, motion, and structure in spaces of any kind, making them indispensable for both theory and practice.</p>
</section>
</section>
<section id="matrix-representation-of-a-linear-map" class="level3">
<h3 class="anchored" data-anchor-id="matrix-representation-of-a-linear-map">42. Matrix Representation of a Linear Map</h3>
<p>Every linear transformation can be expressed concretely as a matrix. This is one of the most powerful bridges in mathematics: it translates abstract functional rules into arrays of numbers that can be calculated, manipulated, and visualized.</p>
<section id="from-abstract-rule-to-concrete-numbers" class="level4">
<h4 class="anchored" data-anchor-id="from-abstract-rule-to-concrete-numbers">From Abstract Rule to Concrete Numbers</h4>
<p>Suppose <span class="math inline">\(T: V \to W\)</span> is a linear transformation between two finite-dimensional vector spaces. To represent <span class="math inline">\(T\)</span> as a matrix, we first select bases:</p>
<ul>
<li><span class="math inline">\(B = \{v_1, v_2, \dots, v_n\}\)</span> for the domain <span class="math inline">\(V\)</span>.</li>
<li><span class="math inline">\(C = \{w_1, w_2, \dots, w_m\}\)</span> for the codomain <span class="math inline">\(W\)</span>.</li>
</ul>
<p>For each basis vector <span class="math inline">\(v_j\)</span>, compute <span class="math inline">\(T(v_j)\)</span>. Each image <span class="math inline">\(T(v_j)\)</span> is a vector in <span class="math inline">\(W\)</span>, so it can be written as a combination of the basis <span class="math inline">\(C\)</span>:</p>
<p><span class="math display">\[
T(v_j) = a_{1j}w_1 + a_{2j}w_2 + \dots + a_{mj}w_m.
\]</span></p>
<p>The coefficients <span class="math inline">\((a_{1j}, a_{2j}, \dots, a_{mj})\)</span> become the j-th column of the matrix representing <span class="math inline">\(T\)</span>.</p>
<p>Thus, the matrix of <span class="math inline">\(T\)</span> relative to bases <span class="math inline">\(B\)</span> and <span class="math inline">\(C\)</span> is</p>
<p><span class="math display">\[
[T]_{B \to C} = \begin{bmatrix} a_{11} &amp; a_{12} &amp; \dots &amp; a_{1n} \\ a_{21} &amp; a_{22} &amp; \dots &amp; a_{2n} \\ \vdots &amp; \vdots &amp; \ddots &amp; \vdots \\ a_{m1} &amp; a_{m2} &amp; \dots &amp; a_{mn} \end{bmatrix}.
\]</span></p>
<p>This guarantees that for any vector <span class="math inline">\(x\)</span> in coordinates relative to <span class="math inline">\(B\)</span>,</p>
<p><span class="math display">\[
[T(x)]_C = [T]_{B \to C}[x]_B.
\]</span></p>
</section>
<section id="standard-basis-case" class="level4">
<h4 class="anchored" data-anchor-id="standard-basis-case">Standard Basis Case</h4>
<p>When both <span class="math inline">\(B\)</span> and <span class="math inline">\(C\)</span> are the standard bases, the process simplifies:</p>
<ul>
<li>Take <span class="math inline">\(T(e_1), T(e_2), \dots, T(e_n)\)</span>.</li>
<li>Place them as columns in a matrix.</li>
</ul>
<p>That matrix directly represents <span class="math inline">\(T\)</span>.</p>
<p>Example: Let <span class="math inline">\(T(x,y) = (2x+y, x-y)\)</span>.</p>
<ul>
<li><span class="math inline">\(T(e_1) = (2,1)\)</span>.</li>
<li><span class="math inline">\(T(e_2) = (1,-1)\)</span>.</li>
</ul>
<p>So the standard matrix is</p>
<p><span class="math display">\[
A = \begin{bmatrix} 2 &amp; 1 \\ 1 &amp; -1 \end{bmatrix}.
\]</span></p>
<p>For any vector <span class="math inline">\(\begin{bmatrix} x \\ y \end{bmatrix}\)</span>,</p>
<p><span class="math display">\[
T(x,y) = A \begin{bmatrix} x \\ y \end{bmatrix}.
\]</span></p>
</section>
<section id="multiple-perspectives" class="level4">
<h4 class="anchored" data-anchor-id="multiple-perspectives">Multiple Perspectives</h4>
<ul>
<li>Columns-as-images: Each column shows where a basis vector goes.</li>
<li>Row view: Each row encodes how to compute one coordinate of the output.</li>
<li>Operator view: The matrix acts like a machine: input vector → multiply → output vector.</li>
</ul>
</section>
<section id="geometric-insight" class="level4">
<h4 class="anchored" data-anchor-id="geometric-insight">Geometric Insight</h4>
<p>Matrices reshape space. In <span class="math inline">\(\mathbb{R}^2\)</span>:</p>
<ul>
<li>The first column shows where the x-axis goes.</li>
<li>The second column shows where the y-axis goes. The entire grid is determined by these two images.</li>
</ul>
<p>In <span class="math inline">\(\mathbb{R}^3\)</span>, the three columns are the images of the unit coordinate directions, defining how the whole space twists, rotates, or compresses.</p>
</section>
<section id="applications-5" class="level4">
<h4 class="anchored" data-anchor-id="applications-5">Applications</h4>
<ol type="1">
<li>Computer graphics: Rotations, scaling, and projections are represented by small matrices.</li>
<li>Robotics: Coordinate changes between joints and workspaces rely on transformation matrices.</li>
<li>Data science: Linear maps such as PCA are implemented with matrices that project data into lower dimensions.</li>
<li>Physics: Linear operators like rotations, boosts, and stress tensors are matrix representations.</li>
</ol>
</section>
<section id="why-it-matters-38" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-38">Why It Matters</h4>
<ol type="1">
<li>Matrices are computational tools: we can add, multiply, invert them.</li>
<li>They let us use algorithms like Gaussian elimination, LU/QR/SVD to study transformations.</li>
<li>They link abstract vector space theory to hands-on numerical calculation.</li>
<li>They reveal the structure of transformations at a glance, just by inspecting columns and rows.</li>
</ol>
</section>
<section id="try-it-yourself-41" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-41">Try It Yourself</h4>
<ol type="1">
<li>Find the matrix for the transformation <span class="math inline">\(T(x,y,z) = (x+2y, y+z, x+z)\)</span> in the standard basis.</li>
<li>Compute the matrix of <span class="math inline">\(T: \mathbb{R}^2 \to \mathbb{R}^2\)</span>, where <span class="math inline">\(T(x,y) = (x-y, x+y)\)</span>.</li>
<li>Using the basis <span class="math inline">\(B=\{(1,1), (1,-1)\}\)</span> for <span class="math inline">\(\mathbb{R}^2\)</span>, find the matrix of <span class="math inline">\(T(x,y) = (2x, y)\)</span> relative to <span class="math inline">\(B\)</span>.</li>
<li>Challenge: Show that matrix multiplication corresponds to composition of transformations, i.e.&nbsp;<span class="math inline">\([S \circ T] = [S][T]\)</span>.</li>
</ol>
<p>Matrix representations are the practical form of linear transformations, turning elegant definitions into something we can compute, visualize, and apply across science and engineering.</p>
</section>
</section>
<section id="kernel-and-image" class="level3">
<h3 class="anchored" data-anchor-id="kernel-and-image">43. Kernel and Image</h3>
<p>Every linear transformation hides two essential structures: the set of vectors that collapse to zero, and the set of all possible outputs. These are called the kernel and the image. They are the DNA of a linear map, revealing its internal structure, its strengths, and its limitations.</p>
<section id="the-kernel" class="level4">
<h4 class="anchored" data-anchor-id="the-kernel">The Kernel</h4>
<p>The kernel (or null space) of a linear transformation <span class="math inline">\(T: V \to W\)</span> is defined as:</p>
<p><span class="math display">\[
\ker(T) = \{ v \in V : T(v) = 0 \}.
\]</span></p>
<ul>
<li>It is the set of all vectors that the transformation sends to the zero vector.</li>
<li>It measures how much information is “lost” under the transformation.</li>
<li>The kernel is always a subspace of the domain <span class="math inline">\(V\)</span>.</li>
</ul>
<p>Examples:</p>
<ol type="1">
<li><p>For <span class="math inline">\(T: \mathbb{R}^2 \to \mathbb{R}^2\)</span>, <span class="math inline">\(T(x,y) = (x,0)\)</span>.</p>
<ul>
<li>Kernel: all vectors of the form <span class="math inline">\((0,y)\)</span>. This is the y-axis.</li>
</ul></li>
<li><p>For <span class="math inline">\(T: \mathbb{R}^3 \to \mathbb{R}^2\)</span>, <span class="math inline">\(T(x,y,z) = (x,y)\)</span>.</p>
<ul>
<li>Kernel: all vectors of the form <span class="math inline">\((0,0,z)\)</span>. This is the z-axis.</li>
</ul></li>
</ol>
<p>The kernel tells us which directions in the domain vanish under <span class="math inline">\(T\)</span>.</p>
</section>
<section id="the-image" class="level4">
<h4 class="anchored" data-anchor-id="the-image">The Image</h4>
<p>The image (or range) of a linear transformation is defined as:</p>
<p><span class="math display">\[
\text{im}(T) = \{ T(v) : v \in V \}.
\]</span></p>
<ul>
<li>It is the set of all vectors that can actually be reached by applying <span class="math inline">\(T\)</span>.</li>
<li>It describes the “output space” of the transformation.</li>
<li>The image is always a subspace of the codomain <span class="math inline">\(W\)</span>.</li>
</ul>
<p>Examples:</p>
<ol type="1">
<li><p>For <span class="math inline">\(T(x,y) = (x,0)\)</span>:</p>
<ul>
<li>Image: all vectors of the form <span class="math inline">\((a,0)\)</span>. This is the x-axis.</li>
</ul></li>
<li><p>For <span class="math inline">\(T(x,y,z) = (x+y, y+z)\)</span>:</p>
<ul>
<li>Image: all of <span class="math inline">\(\mathbb{R}^2\)</span>. Any vector <span class="math inline">\((u,v)\)</span> can be achieved by solving equations for <span class="math inline">\((x,y,z)\)</span>.</li>
</ul></li>
</ol>
</section>
<section id="kernel-and-image-together" class="level4">
<h4 class="anchored" data-anchor-id="kernel-and-image-together">Kernel and Image Together</h4>
<p>These two subspaces reflect two aspects of <span class="math inline">\(T\)</span>:</p>
<ul>
<li>The kernel measures the collapse in dimension.</li>
<li>The image measures the preserved and transmitted directions.</li>
</ul>
<p>A central result is the Rank–Nullity Theorem:</p>
<p><span class="math display">\[
\dim(\ker T) + \dim(\text{im }T) = \dim(V).
\]</span></p>
<ul>
<li><span class="math inline">\(\dim(\ker T)\)</span> is the nullity.</li>
<li><span class="math inline">\(\dim(\text{im }T)\)</span> is the rank.</li>
</ul>
<p>This theorem guarantees a perfect balance: the domain splits into lost directions (kernel) and active directions (image).</p>
</section>
<section id="matrix-view" class="level4">
<h4 class="anchored" data-anchor-id="matrix-view">Matrix View</h4>
<p>For a matrix <span class="math inline">\(A\)</span>, the linear map is <span class="math inline">\(T(x) = Ax\)</span>.</p>
<ul>
<li>The kernel is the solution set of <span class="math inline">\(Ax = 0\)</span>.</li>
<li>The image is the column space of <span class="math inline">\(A\)</span>.</li>
</ul>
<p>Example:</p>
<p><span class="math display">\[
A = \begin{bmatrix} 1 &amp; 2 &amp; 3 \\ 0 &amp; 1 &amp; 1 \end{bmatrix}.
\]</span></p>
<ul>
<li>Image: span of the columns</li>
</ul>
<p><span class="math display">\[
\text{im}(A) = \text{span}\{ (1,0), (2,1), (3,1) \}.
\]</span></p>
<ul>
<li>Kernel: solve</li>
</ul>
<p><span class="math display">\[
\begin{bmatrix} 1 &amp; 2 &amp; 3 \\ 0 &amp; 1 &amp; 1 \end{bmatrix} \begin{bmatrix} x \\ y \\ z \end{bmatrix} = \begin{bmatrix} 0 \\ 0 \end{bmatrix}.
\]</span></p>
<p>This leads to solutions like <span class="math inline">\(x=-y-2z\)</span>. So the kernel is 1-dimensional, the image is 2-dimensional, and the domain (3D) splits as <span class="math inline">\(1+2=3\)</span>.</p>
</section>
<section id="geometric-intuition-3" class="level4">
<h4 class="anchored" data-anchor-id="geometric-intuition-3">Geometric Intuition</h4>
<ul>
<li>The kernel is the set of invisible directions, like shadows disappearing in projection.</li>
<li>The image is the set of all shadows that can appear.</li>
<li>Together they describe projection, flattening, stretching, or collapsing.</li>
</ul>
<p>Example: Projecting <span class="math inline">\(\mathbb{R}^3\)</span> onto the xy-plane:</p>
<ul>
<li>Kernel: the z-axis (all points collapsed to zero height).</li>
<li>Image: the entire xy-plane (all possible shadows).</li>
</ul>
</section>
<section id="applications-6" class="level4">
<h4 class="anchored" data-anchor-id="applications-6">Applications</h4>
<ol type="1">
<li>Solving equations: Kernel describes all solutions to <span class="math inline">\(Ax=0\)</span>. Image describes what right-hand sides <span class="math inline">\(b\)</span> make <span class="math inline">\(Ax=b\)</span> solvable.</li>
<li>Data science: Nullity corresponds to redundant features; rank corresponds to useful independent features.</li>
<li>Physics: In mechanics, symmetries often form the kernel of a transformation, while observable quantities form the image.</li>
<li>Control theory: The kernel and image determine controllability and observability of systems.</li>
</ol>
</section>
<section id="why-it-matters-39" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-39">Why It Matters</h4>
<ol type="1">
<li>Kernel and image classify transformations into invertible or not.</li>
<li>They give a precise language to describe dimension changes.</li>
<li>They are the foundation of rank, nullity, and invertibility.</li>
<li>They generalize far beyond matrices: to polynomials, functions, operators, and differential equations.</li>
</ol>
</section>
<section id="try-it-yourself-42" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-42">Try It Yourself</h4>
<ol type="1">
<li>Compute the kernel and image of <span class="math inline">\(T(x,y,z) = (x+y, y+z)\)</span>.</li>
<li>For the projection <span class="math inline">\(T(x,y,z) = (x,y,0)\)</span>, identify kernel and image.</li>
<li>Show that if the kernel is trivial (<span class="math inline">\(\{0\}\)</span>), then the transformation is injective.</li>
<li>Challenge: Prove the rank–nullity theorem for a <span class="math inline">\(3\times 3\)</span> matrix by working through examples.</li>
</ol>
<p>The kernel and image are the twin lenses through which linear transformations are understood. One tells us what disappears, the other what remains. Together, they give the clearest picture of a transformation’s essence.</p>
</section>
</section>
<section id="invertibility-and-isomorphisms" class="level3">
<h3 class="anchored" data-anchor-id="invertibility-and-isomorphisms">44. Invertibility and Isomorphisms</h3>
<p>Linear transformations come in many forms: some collapse space into lower dimensions, others stretch it, and a special group preserves all information perfectly. These special transformations are invertible, meaning they can be reversed exactly. When two vector spaces are related by such a transformation, we say they are isomorphic-structurally identical, even if they look different on the surface.</p>
<section id="invertibility-of-linear-transformations" class="level4">
<h4 class="anchored" data-anchor-id="invertibility-of-linear-transformations">Invertibility of Linear Transformations</h4>
<p>A linear transformation <span class="math inline">\(T: V \to W\)</span> is invertible if there exists another linear transformation <span class="math inline">\(S: W \to V\)</span> such that:</p>
<p><span class="math display">\[
S \circ T = I_V \quad \text{and} \quad T \circ S = I_W,
\]</span></p>
<p>where <span class="math inline">\(I_V\)</span> and <span class="math inline">\(I_W\)</span> are identity maps on <span class="math inline">\(V\)</span> and <span class="math inline">\(W\)</span>.</p>
<ul>
<li><span class="math inline">\(S\)</span> is called the inverse of <span class="math inline">\(T\)</span>.</li>
<li>If such an inverse exists, <span class="math inline">\(T\)</span> is a bijection: both one-to-one (injective) and onto (surjective).</li>
<li>In finite-dimensional spaces, this is equivalent to saying that <span class="math inline">\(T\)</span> is represented by an invertible matrix.</li>
</ul>
</section>
<section id="invertible-matrices" class="level4">
<h4 class="anchored" data-anchor-id="invertible-matrices">Invertible Matrices</h4>
<p>An <span class="math inline">\(n \times n\)</span> matrix <span class="math inline">\(A\)</span> is invertible if there exists another <span class="math inline">\(n \times n\)</span> matrix <span class="math inline">\(A^{-1}\)</span> such that:</p>
<p><span class="math display">\[
AA^{-1} = A^{-1}A = I.
\]</span></p>
<p>Characterizations of Invertibility:</p>
<ol type="1">
<li><span class="math inline">\(A\)</span> is invertible ⇔ <span class="math inline">\(\det(A) \neq 0\)</span>.</li>
<li>⇔ Columns of <span class="math inline">\(A\)</span> are linearly independent.</li>
<li>⇔ Columns of <span class="math inline">\(A\)</span> span <span class="math inline">\(\mathbb{R}^n\)</span>.</li>
<li>⇔ Rank of <span class="math inline">\(A\)</span> is <span class="math inline">\(n\)</span>.</li>
<li>⇔ The system <span class="math inline">\(Ax=b\)</span> has exactly one solution for every <span class="math inline">\(b\)</span>.</li>
</ol>
<p>All these properties tie together: invertibility means no information is lost when transforming vectors.</p>
</section>
<section id="isomorphisms-of-vector-spaces" class="level4">
<h4 class="anchored" data-anchor-id="isomorphisms-of-vector-spaces">Isomorphisms of Vector Spaces</h4>
<p>Two vector spaces <span class="math inline">\(V\)</span> and <span class="math inline">\(W\)</span> are isomorphic if there exists a bijective linear transformation <span class="math inline">\(T: V \to W\)</span>.</p>
<ul>
<li><p>This means <span class="math inline">\(V\)</span> and <span class="math inline">\(W\)</span> are “the same” in structure, though they may look different.</p></li>
<li><p>For finite-dimensional spaces:</p>
<p><span class="math display">\[
V \cong W \quad \text{if and only if} \quad \dim(V) = \dim(W).
\]</span></p></li>
<li><p>Example: <span class="math inline">\(\mathbb{R}^2\)</span> and the set of all polynomials of degree ≤ 1 are isomorphic, because both have dimension 2.</p></li>
</ul>
</section>
<section id="examples-of-invertibility" class="level4">
<h4 class="anchored" data-anchor-id="examples-of-invertibility">Examples of Invertibility</h4>
<ol type="1">
<li><p>Rotation in the plane: Every rotation matrix has an inverse (rotation by the opposite angle).</p>
<p><span class="math display">\[
R(\theta) = \begin{bmatrix} \cos\theta &amp; -\sin\theta \\ \sin\theta &amp; \cos\theta \end{bmatrix}, \quad R(\theta)^{-1} = R(-\theta).
\]</span></p></li>
<li><p>Scaling by nonzero factor: <span class="math inline">\(T(x) = ax\)</span> with <span class="math inline">\(a \neq 0\)</span>. Inverse is <span class="math inline">\(T^{-1}(x) = \tfrac{1}{a}x\)</span>.</p></li>
<li><p>Projection onto a line: Not invertible, because depth is lost. The kernel is nontrivial.</p></li>
<li><p>Differentiation on polynomials of degree ≤ n: Not invertible, since constant terms vanish in the kernel.</p></li>
<li><p>Differentiation on exponential functions: Invertible: the inverse is integration (up to constants).</p></li>
</ol>
</section>
<section id="geometric-interpretation-8" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-8">Geometric Interpretation</h4>
<ul>
<li>Invertible transformations preserve dimension: no flattening or collapsing occurs.</li>
<li>They may rotate, shear, stretch, or reflect, but every input vector can be uniquely recovered.</li>
<li>The determinant tells the “volume scaling” of the transformation: invertibility requires this volume not to collapse to zero.</li>
</ul>
</section>
<section id="applications-7" class="level4">
<h4 class="anchored" data-anchor-id="applications-7">Applications</h4>
<ol type="1">
<li>Computer graphics: Invertible matrices allow smooth transformations where no information is lost. Non-invertible maps (like projections) create 2D renderings from 3D worlds.</li>
<li>Cryptography: Encryption systems rely on invertible linear maps for encoding/decoding.</li>
<li>Robotics: Transformations between joint and workspace coordinates must often be invertible for precise control.</li>
<li>Data science: PCA often reduces dimension (non-invertible), but whitening transformations are invertible within the chosen subspace.</li>
<li>Physics: Coordinate changes (e.g., Galilean or Lorentz transformations) are invertible, ensuring that physical laws remain consistent.</li>
</ol>
</section>
<section id="why-it-matters-40" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-40">Why It Matters</h4>
<ol type="1">
<li>Invertible maps preserve the entire structure of a vector space.</li>
<li>They classify vector spaces: if two have the same dimension, they are fundamentally the same via isomorphism.</li>
<li>They allow reversible modeling, essential in physics, cryptography, and computation.</li>
<li>They highlight the delicate balance between lossless transformations (invertible) and lossy ones (non-invertible).</li>
</ol>
</section>
<section id="try-it-yourself-43" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-43">Try It Yourself</h4>
<ol type="1">
<li>Prove that the matrix <span class="math inline">\(\begin{bmatrix} 2 &amp; 1 \\ 3 &amp; 2 \end{bmatrix}\)</span> is invertible by computing its determinant and its inverse.</li>
<li>Show that projection onto the x-axis in <span class="math inline">\(\mathbb{R}^2\)</span> is not invertible. Identify its kernel.</li>
<li>Construct an explicit isomorphism between <span class="math inline">\(\mathbb{R}^3\)</span> and the space of polynomials of degree ≤ 2.</li>
<li>Challenge: Prove that if <span class="math inline">\(T\)</span> is an isomorphism, then it maps bases to bases.</li>
</ol>
<p>Invertibility and isomorphism are the gateways from “linear rules” to the grand idea of equivalence. They allow us to say, with mathematical precision, when two spaces are truly the same in structure-different clothes, same skeleton.</p>
</section>
</section>
<section id="composition-powers-and-iteration" class="level3">
<h3 class="anchored" data-anchor-id="composition-powers-and-iteration">45. Composition, Powers, and Iteration</h3>
<p>Linear transformations are not isolated operations-they can be combined, repeated, and layered to build more complex effects. This leads us to the ideas of composition, powers of transformations, and iteration. These concepts form the backbone of linear dynamics, algorithms, and many real-world systems where repeated actions accumulate into surprising results.</p>
<section id="composition-of-linear-transformations" class="level4">
<h4 class="anchored" data-anchor-id="composition-of-linear-transformations">Composition of Linear Transformations</h4>
<p>If <span class="math inline">\(T: U \to V\)</span> and <span class="math inline">\(S: V \to W\)</span> are linear transformations, then their composition is another transformation</p>
<p><span class="math display">\[
S \circ T : U \to W, \quad (S \circ T)(u) = S(T(u)).
\]</span></p>
<ul>
<li><p>Composition is associative: <span class="math inline">\((R \circ S) \circ T = R \circ (S \circ T)\)</span>.</p></li>
<li><p>Composition is linear: the result of composing two linear maps is still linear.</p></li>
<li><p>In terms of matrices, if <span class="math inline">\(T(x) = Ax\)</span> and <span class="math inline">\(S(x) = Bx\)</span>, then</p>
<p><span class="math display">\[
(S \circ T)(x) = B(Ax) = (BA)x.
\]</span></p>
<p>Notice that the order matters: composition corresponds to matrix multiplication.</p></li>
</ul>
<p>Example:</p>
<ol type="1">
<li><span class="math inline">\(T(x,y) = (x+2y, y)\)</span>.</li>
<li><span class="math inline">\(S(x,y) = (2x, x-y)\)</span>. Then <span class="math inline">\((S \circ T)(x,y) = S(x+2y,y) = (2(x+2y), (x+2y)-y) = (2x+4y, x+y)\)</span>. Matrix multiplication confirms the same result.</li>
</ol>
</section>
<section id="powers-of-transformations" class="level4">
<h4 class="anchored" data-anchor-id="powers-of-transformations">Powers of Transformations</h4>
<p>If <span class="math inline">\(T: V \to V\)</span>, we can apply it repeatedly:</p>
<p><span class="math display">\[
T^2 = T \circ T, \quad T^3 = T \circ T \circ T, \quad \dots
\]</span></p>
<ul>
<li>These are called powers of <span class="math inline">\(T\)</span>.</li>
<li>If <span class="math inline">\(T(x) = Ax\)</span>, then <span class="math inline">\(T^k(x) = A^k x\)</span>.</li>
<li>Powers of transformations capture repeated processes, like compounding interest, population growth, or iterative algorithms.</li>
</ul>
<p>Example: Let <span class="math inline">\(T(x,y) = (2x, 3y)\)</span>. Then</p>
<p><span class="math display">\[
T^n(x,y) = (2^n x, 3^n y).
\]</span></p>
<p>Each iteration amplifies the scaling along different directions.</p>
</section>
<section id="iteration-and-dynamical-systems" class="level4">
<h4 class="anchored" data-anchor-id="iteration-and-dynamical-systems">Iteration and Dynamical Systems</h4>
<p>Iteration means applying the same transformation repeatedly to study long-term behavior:</p>
<p><span class="math display">\[
x_{k+1} = T(x_k), \quad x_0 \text{ given}.
\]</span></p>
<ul>
<li>This creates a discrete dynamical system.</li>
<li>Depending on <span class="math inline">\(T\)</span>, vectors may grow, shrink, oscillate, or stabilize.</li>
</ul>
<p>Example 1 (Markov Chains): If <span class="math inline">\(T\)</span> is a stochastic matrix, iteration describes probability evolution over time. Eventually, the system may converge to a steady-state distribution.</p>
<p>Example 2 (Population Models): If <span class="math inline">\(T\)</span> describes how sub-populations interact, iteration simulates generations. Eigenvalues dictate whether populations explode, stabilize, or vanish.</p>
<p>Example 3 (Computer Graphics): Repeated affine transformations create fractals like the Sierpinski triangle.</p>
</section>
<section id="stability-and-eigenvalues" class="level4">
<h4 class="anchored" data-anchor-id="stability-and-eigenvalues">Stability and Eigenvalues</h4>
<p>The behavior of <span class="math inline">\(T^n(x)\)</span> depends heavily on eigenvalues of the transformation.</p>
<ul>
<li>If <span class="math inline">\(|\lambda| &lt; 1\)</span>, repeated application shrinks vectors in that direction to zero.</li>
<li>If <span class="math inline">\(|\lambda| &gt; 1\)</span>, repeated application causes exponential growth.</li>
<li>If <span class="math inline">\(|\lambda| = 1\)</span>, vectors rotate or oscillate without changing length.</li>
</ul>
<p>This link between powers and eigenvalues underpins many algorithms in numerical analysis and physics.</p>
</section>
<section id="geometric-interpretation-9" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-9">Geometric Interpretation</h4>
<ul>
<li>Composition = chaining geometric actions (rotate then reflect, scale then shear).</li>
<li>Powers = applying the same action repeatedly (rotating 90° four times = identity).</li>
<li>Iteration = exploring the “orbit” of a vector under repeated transformations.</li>
</ul>
</section>
<section id="applications-8" class="level4">
<h4 class="anchored" data-anchor-id="applications-8">Applications</h4>
<ol type="1">
<li>Search engines: PageRank is computed by iterating a linear transformation until it stabilizes.</li>
<li>Economics: Input–output models iterate to predict long-term equilibrium of industries.</li>
<li>Physics: Time evolution of quantum states is modeled by repeated application of unitary operators.</li>
<li>Numerical methods: Iterative solvers (like power iteration) approximate eigenvectors.</li>
<li>Computer graphics: Iterated function systems generate self-similar fractals.</li>
</ol>
</section>
<section id="why-it-matters-41" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-41">Why It Matters</h4>
<ol type="1">
<li>Composition unifies matrix multiplication and transformation chaining.</li>
<li>Powers reveal exponential growth, decay, and oscillation.</li>
<li>Iteration is the core of modeling dynamic processes in mathematics, science, and engineering.</li>
<li>The link to eigenvalues makes these ideas the foundation of stability analysis.</li>
</ol>
</section>
<section id="try-it-yourself-44" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-44">Try It Yourself</h4>
<ol type="1">
<li>Let <span class="math inline">\(T(x,y) = (x+y, y)\)</span>. Compute <span class="math inline">\(T^2(x,y)\)</span> and <span class="math inline">\(T^3(x,y)\)</span>. What happens as <span class="math inline">\(n \to \infty\)</span>?</li>
<li>Consider rotation by 90° in <span class="math inline">\(\mathbb{R}^2\)</span>. Show that <span class="math inline">\(T^4 = I\)</span>.</li>
<li>For matrix <span class="math inline">\(A = \begin{bmatrix} 0.5 &amp; 0.5 \\ 0.5 &amp; 0.5 \end{bmatrix}\)</span>, iterate <span class="math inline">\(A^n\)</span>. What happens to arbitrary vectors?</li>
<li>Challenge: Prove that if <span class="math inline">\(A\)</span> is diagonalizable as <span class="math inline">\(A = PDP^{-1}\)</span>, then <span class="math inline">\(A^n = PD^nP^{-1}\)</span>. Use this to analyze long-term behavior.</li>
</ol>
<p>Composition, powers, and iteration take linear algebra beyond static equations into the world of processes over time. They explain how small, repeated steps shape long-term outcomes-whether stabilizing systems, amplifying signals, or creating infinite complexity.</p>
</section>
</section>
<section id="similarity-and-conjugation" class="level3">
<h3 class="anchored" data-anchor-id="similarity-and-conjugation">46. Similarity and Conjugation</h3>
<p>In linear algebra, different matrices can represent the same underlying transformation when written in different coordinate systems. This relationship is captured by the idea of similarity. Two matrices are similar if one is obtained from the other by a conjugation with an invertible change-of-basis matrix. This concept is central to understanding canonical forms, eigenvalue decompositions, and the deep structure of linear operators.</p>
<section id="definition-of-similarity" class="level4">
<h4 class="anchored" data-anchor-id="definition-of-similarity">Definition of Similarity</h4>
<p>Two <span class="math inline">\(n \times n\)</span> matrices <span class="math inline">\(A\)</span> and <span class="math inline">\(B\)</span> are called similar if there exists an invertible matrix <span class="math inline">\(P\)</span> such that:</p>
<p><span class="math display">\[
B = P^{-1}AP.
\]</span></p>
<ul>
<li>Here, <span class="math inline">\(P\)</span> represents a change of basis.</li>
<li><span class="math inline">\(A\)</span> and <span class="math inline">\(B\)</span> describe the same linear transformation, but expressed relative to different bases.</li>
</ul>
</section>
<section id="conjugation-as-change-of-basis" class="level4">
<h4 class="anchored" data-anchor-id="conjugation-as-change-of-basis">Conjugation as Change of Basis</h4>
<p>Suppose <span class="math inline">\(T: V \to V\)</span> is a linear transformation and <span class="math inline">\(A\)</span> is its matrix in basis <span class="math inline">\(B\)</span>. If we switch to a new basis <span class="math inline">\(C\)</span>, the matrix becomes <span class="math inline">\(B\)</span>. The conversion is:</p>
<p><span class="math display">\[
B = P^{-1}AP,
\]</span></p>
<p>where <span class="math inline">\(P\)</span> is the change-of-basis matrix from basis <span class="math inline">\(B\)</span> to basis <span class="math inline">\(C\)</span>.</p>
<p>This shows that similarity is not just algebraic coincidence-it’s geometric: the operator is the same, but our perspective (basis) has changed.</p>
</section>
<section id="properties-preserved-under-similarity" class="level4">
<h4 class="anchored" data-anchor-id="properties-preserved-under-similarity">Properties Preserved Under Similarity</h4>
<p>If <span class="math inline">\(A\)</span> and <span class="math inline">\(B\)</span> are similar, they share many key properties:</p>
<ol type="1">
<li>Determinant: <span class="math inline">\(\det(A) = \det(B)\)</span>.</li>
<li>Trace: <span class="math inline">\(\text{tr}(A) = \text{tr}(B)\)</span>.</li>
<li>Rank: <span class="math inline">\(\text{rank}(A) = \text{rank}(B)\)</span>.</li>
<li>Eigenvalues: Same set of eigenvalues (with multiplicity).</li>
<li>Characteristic polynomial: Identical.</li>
<li>Minimal polynomial: Identical.</li>
</ol>
<p>These invariants define the “skeleton” of a linear operator, unaffected by coordinate changes.</p>
</section>
<section id="examples-2" class="level4">
<h4 class="anchored" data-anchor-id="examples-2">Examples</h4>
<ol type="1">
<li><p>Rotation in the plane: The matrix for rotation by 90° is</p>
<p><span class="math display">\[
A = \begin{bmatrix} 0 &amp; -1 \\ 1 &amp; 0 \end{bmatrix}.
\]</span></p>
<p>In another basis, the rotation might be represented by a more complicated-looking matrix, but all such matrices are similar to <span class="math inline">\(A\)</span>.</p></li>
<li><p>Diagonalization: A matrix <span class="math inline">\(A\)</span> is diagonalizable if it is similar to a diagonal matrix <span class="math inline">\(D\)</span>. That is,</p>
<p><span class="math display">\[
A = PDP^{-1}.
\]</span></p>
<p>Here, similarity reduces <span class="math inline">\(A\)</span> to its simplest form.</p></li>
<li><p>Shear transformation: A shear matrix <span class="math inline">\(\begin{bmatrix} 1 &amp; 1 \\ 0 &amp; 1 \end{bmatrix}\)</span> is not diagonalizable, but it may be similar to a Jordan block.</p></li>
</ol>
</section>
<section id="geometric-interpretation-10" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-10">Geometric Interpretation</h4>
<ul>
<li>Similarity says: two matrices may look different, but they are “the same” transformation seen from different coordinate systems.</li>
<li>Conjugation is the mathematical act of relabeling coordinates.</li>
<li>Think of shifting your camera angle: the scene hasn’t changed, only the perspective has.</li>
</ul>
</section>
<section id="applications-9" class="level4">
<h4 class="anchored" data-anchor-id="applications-9">Applications</h4>
<ol type="1">
<li>Diagonalization: Reducing a matrix to diagonal form (when possible) uses similarity. This simplifies powers, exponentials, and iterative analysis.</li>
<li>Jordan canonical form: Every square matrix is similar to a Jordan form, giving a complete structural classification.</li>
<li>Quantum mechanics: Operators on state spaces often change representation, but similarity guarantees invariance of spectra.</li>
<li>Control theory: Canonical forms simplify analysis of system stability and controllability.</li>
<li>Numerical methods: Eigenvalue algorithms rely on repeated similarity transformations (e.g., QR algorithm).</li>
</ol>
</section>
<section id="why-it-matters-42" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-42">Why It Matters</h4>
<ol type="1">
<li>Similarity reveals the true identity of a linear operator, independent of coordinates.</li>
<li>It allows simplification: many problems become easier in the right basis.</li>
<li>It preserves invariants, giving us tools to classify and compare operators.</li>
<li>It connects abstract algebra with concrete computations in geometry, physics, and engineering.</li>
</ol>
</section>
<section id="try-it-yourself-45" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-45">Try It Yourself</h4>
<ol type="1">
<li>Show that <span class="math inline">\(\begin{bmatrix} 2 &amp; 1 \\ 0 &amp; 2 \end{bmatrix}\)</span> is similar to <span class="math inline">\(\begin{bmatrix} 2 &amp; 0 \\ 0 &amp; 2 \end{bmatrix}\)</span>. Why or why not?</li>
<li>Compute <span class="math inline">\(P^{-1}AP\)</span> for <span class="math inline">\(A = \begin{bmatrix} 1 &amp; 2 \\ 0 &amp; 1 \end{bmatrix}\)</span> and <span class="math inline">\(P = \begin{bmatrix} 1 &amp; 1 \\ 0 &amp; 1 \end{bmatrix}\)</span>. Interpret the result.</li>
<li>Prove that if two matrices are similar, they must have the same trace.</li>
<li>Challenge: Show that if <span class="math inline">\(A\)</span> and <span class="math inline">\(B\)</span> are similar, then <span class="math inline">\(A^k\)</span> and <span class="math inline">\(B^k\)</span> are also similar for all integers <span class="math inline">\(k \geq 0\)</span>.</li>
</ol>
<p>Similarity and conjugation elevate linear algebra from mere calculation to structural understanding. They tell us when two seemingly different matrices are just different “faces” of the same underlying transformation.</p>
</section>
</section>
<section id="projections-and-reflections" class="level3">
<h3 class="anchored" data-anchor-id="projections-and-reflections">47. Projections and Reflections</h3>
<p>Among the many transformations in linear algebra, two stand out for their geometric clarity and practical importance: projections and reflections. These operations reshape vectors in simple but powerful ways, and they form the building blocks of algorithms in statistics, optimization, graphics, and physics.</p>
<section id="projection-flattening-onto-a-subspace" class="level4">
<h4 class="anchored" data-anchor-id="projection-flattening-onto-a-subspace">Projection: Flattening onto a Subspace</h4>
<p>A projection is a linear transformation that takes a vector and drops it onto a subspace, like casting a shadow.</p>
<p>Formally, if <span class="math inline">\(W\)</span> is a subspace of <span class="math inline">\(V\)</span>, the projection of a vector <span class="math inline">\(v\)</span> onto <span class="math inline">\(W\)</span> is the unique vector <span class="math inline">\(w \in W\)</span> that is closest to <span class="math inline">\(v\)</span>.</p>
<p>In <span class="math inline">\(\mathbb{R}^2\)</span>: projecting onto the x-axis takes <span class="math inline">\((x,y)\)</span> and produces <span class="math inline">\((x,0)\)</span>.</p>
<section id="orthogonal-projection-formula" class="level5">
<h5 class="anchored" data-anchor-id="orthogonal-projection-formula">Orthogonal Projection Formula</h5>
<p>Suppose <span class="math inline">\(u\)</span> is a nonzero vector. The projection of <span class="math inline">\(v\)</span> onto the line spanned by <span class="math inline">\(u\)</span> is:</p>
<p><span class="math display">\[
\text{proj}_u(v) = \frac{v \cdot u}{u \cdot u} u.
\]</span></p>
<p>This formula works in any dimension. It uses the dot product to measure how much of <span class="math inline">\(v\)</span> points in the direction of <span class="math inline">\(u\)</span>.</p>
<p>Example: Project <span class="math inline">\((2,3)\)</span> onto <span class="math inline">\(u=(1,1)\)</span>:</p>
<p><span class="math display">\[
\text{proj}_u(2,3) = \frac{(2,3)\cdot(1,1)}{(1,1)\cdot(1,1)} (1,1) = \frac{5}{2}(1,1) = (2.5,2.5).
\]</span></p>
<p>The vector <span class="math inline">\((2,3)\)</span> splits into <span class="math inline">\((2.5,2.5)\)</span> along the line plus <span class="math inline">\((-0.5,0.5)\)</span> orthogonal to it.</p>
</section>
<section id="projection-matrices" class="level5">
<h5 class="anchored" data-anchor-id="projection-matrices">Projection Matrices</h5>
<p>For unit vector <span class="math inline">\(u\)</span>:</p>
<p><span class="math display">\[
P = uu^T
\]</span></p>
<p>is the projection matrix onto the span of <span class="math inline">\(u\)</span>.</p>
<p>For a general subspace with orthonormal basis columns in matrix <span class="math inline">\(Q\)</span>:</p>
<p><span class="math display">\[
P = QQ^T
\]</span></p>
<p>projects any vector onto that subspace.</p>
<p>Properties:</p>
<ol type="1">
<li><span class="math inline">\(P^2 = P\)</span> (idempotent).</li>
<li><span class="math inline">\(P^T = P\)</span> (symmetric, for orthogonal projections).</li>
</ol>
</section>
</section>
<section id="reflection-flipping-across-a-subspace" class="level4">
<h4 class="anchored" data-anchor-id="reflection-flipping-across-a-subspace">Reflection: Flipping Across a Subspace</h4>
<p>A reflection takes a vector and flips it across a line or plane. Geometrically, it’s like a mirror.</p>
<p>Reflection across a line spanned by unit vector <span class="math inline">\(u\)</span>:</p>
<p><span class="math display">\[
R(v) = 2\text{proj}_u(v) - v.
\]</span></p>
<p>Matrix form:</p>
<p><span class="math display">\[
R = 2uu^T - I.
\]</span></p>
<p>Example: Reflect <span class="math inline">\((2,3)\)</span> across the line <span class="math inline">\(y=x\)</span>. With <span class="math inline">\(u=(1/\sqrt{2},1/\sqrt{2})\)</span>:</p>
<p><span class="math display">\[
R = \begin{bmatrix} 0 &amp; 1 \\ 1 &amp; 0 \end{bmatrix}.
\]</span></p>
<p>So reflection swaps coordinates: <span class="math inline">\((2,3) \mapsto (3,2)\)</span>.</p>
</section>
<section id="geometric-insight-1" class="level4">
<h4 class="anchored" data-anchor-id="geometric-insight-1">Geometric Insight</h4>
<ul>
<li>Projection shortens vectors by removing components orthogonal to the subspace.</li>
<li>Reflection preserves length but flips orientation relative to the subspace.</li>
<li>Projection is about approximation (“closest point”), reflection is about symmetry.</li>
</ul>
</section>
<section id="applications-10" class="level4">
<h4 class="anchored" data-anchor-id="applications-10">Applications</h4>
<ol type="1">
<li>Statistics &amp; Machine Learning: Least-squares regression is projection of data onto the span of predictor variables.</li>
<li>Computer Graphics: Projection transforms 3D scenes into 2D screen images. Reflections simulate mirrors and shiny surfaces.</li>
<li>Optimization: Projections enforce constraints by bringing guesses back into feasible regions.</li>
<li>Physics: Reflections describe wave behavior, optics, and particle interactions.</li>
<li>Numerical Methods: Projection operators are key to iterative algorithms (like Krylov subspace methods).</li>
</ol>
</section>
<section id="why-it-matters-43" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-43">Why It Matters</h4>
<ol type="1">
<li>Projection captures the essence of approximation: keeping what fits, discarding what doesn’t.</li>
<li>Reflection embodies symmetry and invariance, key to geometry and physics.</li>
<li>Both are linear, with elegant matrix representations.</li>
<li>They combine easily with other transformations, making them versatile in computation.</li>
</ol>
</section>
<section id="try-it-yourself-46" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-46">Try It Yourself</h4>
<ol type="1">
<li>Find the projection matrix onto the line spanned by <span class="math inline">\((3,4)\)</span>. Verify it is idempotent.</li>
<li>Compute the reflection of <span class="math inline">\((1,2)\)</span> across the x-axis.</li>
<li>Show that reflection matrices are orthogonal (<span class="math inline">\(R^T R = I\)</span>).</li>
<li>Challenge: For subspace <span class="math inline">\(W\)</span> with orthonormal basis <span class="math inline">\(Q\)</span>, derive the reflection matrix <span class="math inline">\(R = 2QQ^T - I\)</span>.</li>
</ol>
<p>Projections and reflections are two of the purest examples of how linear transformations embody geometric ideas. One approximates, the other symmetrizes-but both expose the deep structure of space through the lens of linear algebra.</p>
</section>
</section>
<section id="rotations-and-shear" class="level3">
<h3 class="anchored" data-anchor-id="rotations-and-shear">48. Rotations and Shear</h3>
<p>Linear transformations can twist, turn, and distort space in strikingly different ways. Two of the most fundamental examples are rotations-which preserve lengths and angles while turning vectors-and shears-which slide one part of space relative to another, distorting shape while often preserving area. These two transformations form the geometric heart of linear algebra, and they are indispensable in graphics, physics, and engineering.</p>
<section id="rotations-in-the-plane" class="level4">
<h4 class="anchored" data-anchor-id="rotations-in-the-plane">Rotations in the Plane</h4>
<p>A rotation in <span class="math inline">\(\mathbb{R}^2\)</span> by an angle <span class="math inline">\(\theta\)</span> is defined as:</p>
<p><span class="math display">\[
R_\theta = \begin{bmatrix} \cos\theta &amp; -\sin\theta \\ \sin\theta &amp; \cos\theta \end{bmatrix}.
\]</span></p>
<p>For any vector <span class="math inline">\((x,y)\)</span>:</p>
<p><span class="math display">\[
R_\theta \begin{bmatrix} x \\ y \end{bmatrix} = \begin{bmatrix} x\cos\theta - y\sin\theta \\ x\sin\theta + y\cos\theta \end{bmatrix}.
\]</span></p>
<p>Properties:</p>
<ol type="1">
<li>Preserves lengths: <span class="math inline">\(\|R_\theta v\| = \|v\|\)</span>.</li>
<li>Preserves angles: the dot product is unchanged.</li>
<li>Determinant = <span class="math inline">\(+1\)</span>, so it preserves orientation and area.</li>
<li>Inverse: <span class="math inline">\(R_\theta^{-1} = R_{-\theta}\)</span>.</li>
</ol>
<p>Example: A 90° rotation:</p>
<p><span class="math display">\[
R_{90^\circ} = \begin{bmatrix} 0 &amp; -1 \\ 1 &amp; 0 \end{bmatrix}, \quad (1,0) \mapsto (0,1).
\]</span></p>
</section>
<section id="rotations-in-three-dimensions" class="level4">
<h4 class="anchored" data-anchor-id="rotations-in-three-dimensions">Rotations in Three Dimensions</h4>
<p>Rotations in <span class="math inline">\(\mathbb{R}^3\)</span> occur around an axis. For example, rotation by angle <span class="math inline">\(\theta\)</span> around the z-axis:</p>
<p><span class="math display">\[
R_z(\theta) = \begin{bmatrix} \cos\theta &amp; -\sin\theta &amp; 0 \\ \sin\theta &amp; \cos\theta &amp; 0 \\ 0 &amp; 0 &amp; 1 \end{bmatrix}.
\]</span></p>
<ul>
<li>Leaves the z-axis fixed.</li>
<li>Rotates the xy-plane like a 2D rotation.</li>
</ul>
<p>General rotations in 3D are described by orthogonal matrices with determinant +1, forming the group <span class="math inline">\(SO(3)\)</span>.</p>
</section>
<section id="shear-transformations" class="level4">
<h4 class="anchored" data-anchor-id="shear-transformations">Shear Transformations</h4>
<p>A shear slides one coordinate direction while keeping another fixed, distorting shapes.</p>
<p>In <span class="math inline">\(\mathbb{R}^2\)</span>:</p>
<p><span class="math display">\[
S = \begin{bmatrix} 1 &amp; k \\ 0 &amp; 1 \end{bmatrix} \quad \text{or} \quad \begin{bmatrix} 1 &amp; 0 \\ k &amp; 1 \end{bmatrix}.
\]</span></p>
<ul>
<li>The first form “slides” x-coordinates depending on y.</li>
<li>The second form slides y-coordinates depending on x.</li>
</ul>
<p>Example:</p>
<p><span class="math display">\[
S = \begin{bmatrix} 1 &amp; 1 \\ 0 &amp; 1 \end{bmatrix}, \quad (x,y) \mapsto (x+y, y).
\]</span></p>
<ul>
<li>Squares become parallelograms.</li>
<li>Areas are preserved if <span class="math inline">\(\det(S) = 1\)</span>.</li>
</ul>
<p>In <span class="math inline">\(\mathbb{R}^3\)</span>: shears distort volumes while preserving parallelism of faces.</p>
</section>
<section id="geometric-comparison" class="level4">
<h4 class="anchored" data-anchor-id="geometric-comparison">Geometric Comparison</h4>
<ul>
<li>Rotation: Preserves size and shape exactly, only changes orientation. Circles remain circles.</li>
<li>Shear: Distorts shape but often preserves area (in 2D) or volume (in 3D). Circles become ellipses or slanted figures.</li>
</ul>
<p>Together, rotations and shears can generate a vast variety of linear distortions.</p>
</section>
<section id="applications-11" class="level4">
<h4 class="anchored" data-anchor-id="applications-11">Applications</h4>
<ol type="1">
<li>Computer Graphics: Rotations orient objects; shears simulate perspective.</li>
<li>Engineering: Shear stresses deform materials; rotations model rigid-body motion.</li>
<li>Robotics: Rotations define arm orientation; shears approximate local deformations.</li>
<li>Physics: Rotations are symmetries of space; shears appear in fluid flows and elasticity.</li>
<li>Data Science: Shears represent changes of variables that preserve volume but distort distributions.</li>
</ol>
</section>
<section id="why-it-matters-44" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-44">Why It Matters</h4>
<ol type="1">
<li>Rotations model pure symmetry-no distortion, just reorientation.</li>
<li>Shears show how geometry can be distorted while preserving volume or area.</li>
<li>Both are building blocks: any invertible matrix in <span class="math inline">\(\mathbb{R}^2\)</span> can be factored into rotations, shears, and scalings.</li>
<li>They bridge algebra and geometry, giving visual meaning to abstract matrices.</li>
</ol>
</section>
<section id="try-it-yourself-47" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-47">Try It Yourself</h4>
<ol type="1">
<li>Rotate <span class="math inline">\((1,0)\)</span> by 60° and compute the result explicitly.</li>
<li>Apply the shear <span class="math inline">\(S=\begin{bmatrix} 1 &amp; 2 \\ 0 &amp; 1 \end{bmatrix}\)</span> to the square with vertices <span class="math inline">\((0,0),(1,0),(0,1),(1,1)\)</span>. What shape results?</li>
<li>Show that rotation matrices are orthogonal (<span class="math inline">\(R^TR=I\)</span>).</li>
<li>Challenge: Prove that any area-preserving <span class="math inline">\(2\times2\)</span> matrix with determinant 1 can be decomposed into a product of rotations and shears.</li>
</ol>
<p>Rotations and shears highlight two complementary sides of linear algebra: symmetry versus distortion. Together, they show how transformations can either preserve the essence of space or bend it into new shapes while keeping its structure intact.</p>
</section>
</section>
<section id="rank-and-operator-viewpoint" class="level3">
<h3 class="anchored" data-anchor-id="rank-and-operator-viewpoint">49. Rank and Operator Viewpoint</h3>
<p>The rank of a linear transformation or matrix is one of the most important measures of its power. It captures how many independent directions a transformation preserves, how much information it carries from input to output, and how “full” its action on space is. Thinking of rank not just as a number, but as a description of an operator, gives us a clearer picture of what transformations really do.</p>
<section id="definition-of-rank-1" class="level4">
<h4 class="anchored" data-anchor-id="definition-of-rank-1">Definition of Rank</h4>
<p>For a matrix <span class="math inline">\(A\)</span> representing a linear transformation <span class="math inline">\(T: V \to W\)</span>:</p>
<p><span class="math display">\[
\text{rank}(A) = \dim(\text{im}(A)) = \dim(\text{im}(T)).
\]</span></p>
<p>That is, the rank is the dimension of the image (or column space). It counts the maximum number of linearly independent columns.</p>
</section>
<section id="basic-properties" class="level4">
<h4 class="anchored" data-anchor-id="basic-properties">Basic Properties</h4>
<ol type="1">
<li><p><span class="math inline">\(\text{rank}(A) \leq \min(m,n)\)</span> for an <span class="math inline">\(m \times n\)</span> matrix.</p></li>
<li><p><span class="math inline">\(\text{rank}(A) = \text{rank}(A^T)\)</span>.</p></li>
<li><p>Rank is equal to the number of pivot columns in row-reduced form.</p></li>
<li><p>Rank links directly with nullity via the rank–nullity theorem:</p>
<p><span class="math display">\[
\text{rank}(A) + \text{nullity}(A) = n.
\]</span></p></li>
</ol>
</section>
<section id="operator-perspective" class="level4">
<h4 class="anchored" data-anchor-id="operator-perspective">Operator Perspective</h4>
<p>Instead of focusing on rows and columns, imagine rank as a measure of how much of the domain is transmitted faithfully to the codomain.</p>
<ul>
<li>If rank = full (<span class="math inline">\(n\)</span>), the transformation is injective: nothing collapses.</li>
<li>If rank = dimension of codomain (<span class="math inline">\(m\)</span>), the transformation is surjective: every target vector can be reached.</li>
<li>If rank is smaller, the transformation compresses space: parts of the domain are “invisible” and collapse into the kernel.</li>
</ul>
<p>Example 1 (Projection): Projection from <span class="math inline">\(\mathbb{R}^3\)</span> onto the xy-plane has rank 2. It annihilates the z-direction but preserves two independent directions.</p>
<p>Example 2 (Rotation): Rotation in <span class="math inline">\(\mathbb{R}^2\)</span> has rank 2. No directions are lost.</p>
<p>Example 3 (Zero map): The transformation sending everything to zero has rank 0.</p>
</section>
<section id="geometric-meaning-7" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-7">Geometric Meaning</h4>
<ul>
<li>Rank = number of independent directions preserved.</li>
<li>A rank-1 transformation maps all of space onto a single line.</li>
<li>Rank-2 in <span class="math inline">\(\mathbb{R}^3\)</span> maps space onto a plane.</li>
<li>Rank-full maps space onto its entire dimension without collapse.</li>
</ul>
<p>Visually: rank describes the “dimensional thickness” of the image.</p>
</section>
<section id="rank-and-matrix-factorizations" class="level4">
<h4 class="anchored" data-anchor-id="rank-and-matrix-factorizations">Rank and Matrix Factorizations</h4>
<p>Rank reveals hidden structure:</p>
<ol type="1">
<li>LU factorization: Rank determines the number of nonzero pivots.</li>
<li>QR factorization: Rank controls the number of orthogonal directions.</li>
<li>SVD (Singular Value Decomposition): The number of nonzero singular values equals the rank.</li>
</ol>
<p>SVD in particular gives a geometric operator view: each nonzero singular value corresponds to a preserved dimension, while zeros indicate collapsed directions.</p>
</section>
<section id="rank-in-applications" class="level4">
<h4 class="anchored" data-anchor-id="rank-in-applications">Rank in Applications</h4>
<ol type="1">
<li>Data compression: Low-rank approximations reduce storage (e.g., image compression with SVD).</li>
<li>Statistics: Rank of the design matrix determines identifiability of regression coefficients.</li>
<li>Machine learning: Rank of weight matrices controls expressive power of models.</li>
<li>Control theory: Rank conditions ensure controllability and observability of systems.</li>
<li>Network analysis: Rank of adjacency or Laplacian matrices reflects connectivity of graphs.</li>
</ol>
</section>
<section id="rank-deficiency" class="level4">
<h4 class="anchored" data-anchor-id="rank-deficiency">Rank Deficiency</h4>
<p>If a transformation has less than full rank, it is rank-deficient. This means:</p>
<ul>
<li>Some directions are lost (kernel nontrivial).</li>
<li>Some outputs are unreachable (image smaller than codomain).</li>
<li>Equations <span class="math inline">\(Ax=b\)</span> may be inconsistent or underdetermined.</li>
</ul>
<p>Detecting and handling rank deficiency is crucial in numerical linear algebra, where ill-conditioning can hide in nearly dependent columns.</p>
</section>
<section id="why-it-matters-45" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-45">Why It Matters</h4>
<ol type="1">
<li>Rank measures the true dimensional effect of a transformation.</li>
<li>It distinguishes between full-strength operators and those that collapse information.</li>
<li>It connects row space, column space, image, and kernel under one number.</li>
<li>It underpins algorithms for regression, decomposition, and dimensionality reduction.</li>
</ol>
</section>
<section id="try-it-yourself-48" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-48">Try It Yourself</h4>
<ol type="1">
<li>Find the rank of <span class="math inline">\(\begin{bmatrix} 1 &amp; 2 &amp; 3 \\ 2 &amp; 4 &amp; 6 \end{bmatrix}\)</span>. Why is it less than 2?</li>
<li>Describe geometrically the image of a rank-1 transformation in <span class="math inline">\(\mathbb{R}^3\)</span>.</li>
<li>For a <span class="math inline">\(5 \times 5\)</span> diagonal matrix with diagonal entries <span class="math inline">\((2,0,3,0,5)\)</span>, compute rank and nullity.</li>
<li>Challenge: Show that for any matrix <span class="math inline">\(A\)</span>, the rank equals the number of nonzero singular values of <span class="math inline">\(A\)</span>.</li>
</ol>
<p>Rank tells us not just how many independent vectors survive a transformation, but also how much structure the operator truly preserves. It is the bridge between abstract linear maps and their practical power.</p>
</section>
</section>
<section id="block-matrices-and-block-maps" class="level3">
<h3 class="anchored" data-anchor-id="block-matrices-and-block-maps">50. Block Matrices and Block Maps</h3>
<p>As problems grow in size, matrices become large and difficult to manage element by element. A powerful strategy is to organize matrices into blocks-submatrices grouped together like tiles in a mosaic. This allows us to treat large transformations as compositions of smaller, more understandable ones. Block matrices preserve structure, simplify computations, and reveal deep insights into how transformations act on subspaces.</p>
<section id="what-are-block-matrices" class="level4">
<h4 class="anchored" data-anchor-id="what-are-block-matrices">What Are Block Matrices?</h4>
<p>A block matrix partitions a matrix into rectangular submatrices. Each block is itself a matrix, and the entire matrix can be manipulated using block rules.</p>
<p>Example: a <span class="math inline">\(4 \times 4\)</span> matrix divided into four <span class="math inline">\(2 \times 2\)</span> blocks:</p>
<p><span class="math display">\[
A = \begin{bmatrix}
A_{11} &amp; A_{12} \\
A_{21} &amp; A_{22}
\end{bmatrix},
\]</span></p>
<p>where each <span class="math inline">\(A_{ij}\)</span> is <span class="math inline">\(2 \times 2\)</span>.</p>
<p>Instead of thinking in terms of 16 entries, we work with 4 blocks.</p>
</section>
<section id="block-maps-as-linear-transformations" class="level4">
<h4 class="anchored" data-anchor-id="block-maps-as-linear-transformations">Block Maps as Linear Transformations</h4>
<p>Suppose <span class="math inline">\(V = V_1 \oplus V_2\)</span> is decomposed into two subspaces. A linear map <span class="math inline">\(T: V \to V\)</span> can be described in terms of how it acts on each component. Relative to this decomposition, the matrix of <span class="math inline">\(T\)</span> has block form:</p>
<p><span class="math display">\[
[T] = \begin{bmatrix}
T_{11} &amp; T_{12} \\
T_{21} &amp; T_{22}
\end{bmatrix}.
\]</span></p>
<ul>
<li><span class="math inline">\(T_{11}\)</span>: how <span class="math inline">\(V_1\)</span> maps into itself.</li>
<li><span class="math inline">\(T_{12}\)</span>: how <span class="math inline">\(V_2\)</span> contributes to <span class="math inline">\(V_1\)</span>.</li>
<li><span class="math inline">\(T_{21}\)</span>: how <span class="math inline">\(V_1\)</span> contributes to <span class="math inline">\(V_2\)</span>.</li>
<li><span class="math inline">\(T_{22}\)</span>: how <span class="math inline">\(V_2\)</span> maps into itself.</li>
</ul>
<p>This decomposition highlights how subspaces interact under the transformation.</p>
</section>
<section id="block-matrix-operations" class="level4">
<h4 class="anchored" data-anchor-id="block-matrix-operations">Block Matrix Operations</h4>
<p>Block matrices obey the same rules as normal matrices, but operations are done block by block.</p>
<p>Addition:</p>
<p><span class="math display">\[
\begin{bmatrix} A &amp; B \\ C &amp; D \end{bmatrix} +
\begin{bmatrix} E &amp; F \\ G &amp; H \end{bmatrix} =
\begin{bmatrix} A+E &amp; B+F \\ C+G &amp; D+H \end{bmatrix}.
\]</span></p>
<p>Multiplication:</p>
<p><span class="math display">\[
\begin{bmatrix} A &amp; B \\ C &amp; D \end{bmatrix}
\begin{bmatrix} E &amp; F \\ G &amp; H \end{bmatrix} =
\begin{bmatrix} AE+BG &amp; AF+BH \\ CE+DG &amp; CF+DH \end{bmatrix}.
\]</span></p>
<p>The formulas look like ordinary multiplication, but each term is itself a product of submatrices.</p>
</section>
<section id="special-block-structures" class="level4">
<h4 class="anchored" data-anchor-id="special-block-structures">Special Block Structures</h4>
<ol type="1">
<li><p>Block Diagonal Matrices:</p>
<p><span class="math display">\[
\begin{bmatrix} A &amp; 0 \\ 0 &amp; D \end{bmatrix}.
\]</span></p>
<p>Independent actions on subspaces-no mixing between them.</p></li>
<li><p>Block Upper Triangular:</p>
<p><span class="math display">\[
\begin{bmatrix} A &amp; B \\ 0 &amp; D \end{bmatrix}.
\]</span></p>
<p>Subspace <span class="math inline">\(V_1\)</span> influences <span class="math inline">\(V_2\)</span>, but not vice versa.</p></li>
<li><p>Block Symmetric: If overall matrix is symmetric, so are certain block relationships: <span class="math inline">\(A^T=A, D^T=D, B^T=C\)</span>.</p></li>
</ol>
<p>These structures appear naturally in decomposition and iterative algorithms.</p>
</section>
<section id="block-matrix-inverses" class="level4">
<h4 class="anchored" data-anchor-id="block-matrix-inverses">Block Matrix Inverses</h4>
<p>Some block matrices can be inverted using special formulas. For</p>
<p><span class="math display">\[
M = \begin{bmatrix} A &amp; B \\ C &amp; D \end{bmatrix},
\]</span></p>
<p>if <span class="math inline">\(A\)</span> is invertible, the inverse can be expressed using the Schur complement:</p>
<p><span class="math display">\[
M^{-1} = \begin{bmatrix}
A^{-1} + A^{-1}B(D-CA^{-1}B)^{-1}CA^{-1} &amp; -A^{-1}B(D-CA^{-1}B)^{-1} \\
-(D-CA^{-1}B)^{-1}CA^{-1} &amp; (D-CA^{-1}B)^{-1}
\end{bmatrix}.
\]</span></p>
<p>This formula is central in statistics, optimization, and numerical analysis.</p>
</section>
<section id="geometric-interpretation-11" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-11">Geometric Interpretation</h4>
<ul>
<li>A block diagonal matrix acts like two independent transformations operating side by side.</li>
<li>A block triangular matrix shows a “hierarchy”: one subspace influences the other but not the reverse.</li>
<li>This decomposition mirrors how systems can be separated into smaller interacting parts.</li>
</ul>
</section>
<section id="applications-12" class="level4">
<h4 class="anchored" data-anchor-id="applications-12">Applications</h4>
<ol type="1">
<li>Numerical Linear Algebra: Block operations optimize computation on large sparse matrices.</li>
<li>Control Theory: State-space models are naturally expressed in block form.</li>
<li>Statistics: Partitioned covariance matrices rely on block inversion formulas.</li>
<li>Machine Learning: Neural networks layer transformations, often structured into blocks for efficiency.</li>
<li>Parallel Computing: Block decomposition distributes large matrix problems across processors.</li>
</ol>
</section>
<section id="why-it-matters-46" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-46">Why It Matters</h4>
<ol type="1">
<li>Block matrices turn big problems into manageable smaller ones.</li>
<li>They reflect natural decompositions of systems into interacting parts.</li>
<li>They make explicit the geometry of subspace interactions.</li>
<li>They provide efficient algorithms, especially for large-scale scientific computing.</li>
</ol>
</section>
<section id="try-it-yourself-49" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-49">Try It Yourself</h4>
<ol type="1">
<li><p>Multiply two <span class="math inline">\(4 \times 4\)</span> matrices written as <span class="math inline">\(2 \times 2\)</span> block matrices and confirm the block multiplication rule.</p></li>
<li><p>Write the projection matrix onto a 2D subspace in <span class="math inline">\(\mathbb{R}^4\)</span> using block form.</p></li>
<li><p>Compute the Schur complement of</p>
<p><span class="math display">\[
\begin{bmatrix} 2 &amp; 1 \\ 1 &amp; 3 \end{bmatrix}.
\]</span></p></li>
<li><p>Challenge: Show that the determinant of a block triangular matrix equals the product of the determinants of its diagonal blocks.</p></li>
</ol>
<p>Block matrices and block maps show how complexity can be organized. Instead of drowning in thousands of entries, we see structure, interaction, and hierarchy-revealing how large systems can be built from simple linear pieces.</p>
</section>
<section id="closing-4" class="level4">
<h4 class="anchored" data-anchor-id="closing-4">Closing</h4>
<pre><code>Shadows twist and turn,
kernels hide and images flow,
form remains within.</code></pre>
</section>
</section>
</section>
<section id="chapter-6.-determinants-and-volume" class="level2">
<h2 class="anchored" data-anchor-id="chapter-6.-determinants-and-volume">Chapter 6. Determinants and volume</h2>
<section id="opening-4" class="level4">
<h4 class="anchored" data-anchor-id="opening-4">Opening</h4>
<pre><code>Areas unfold,
parallels stretch into waves,
scale whispers in signs.</code></pre>
</section>
<section id="areas-volumes-and-signed-scale-factors" class="level3">
<h3 class="anchored" data-anchor-id="areas-volumes-and-signed-scale-factors">51. Areas, Volumes, and Signed Scale Factors</h3>
<p>Determinants often feel like an abstract formula until we see their geometric meaning: they measure area in 2D, volume in 3D, and, in higher dimensions, the general “size” of a transformed shape. Even more, determinants encode whether orientation is preserved or flipped, giving them a “signed” interpretation. This perspective transforms determinants from algebraic curiosities into geometric tools.</p>
<section id="transformations-and-scaling-of-space" class="level4">
<h4 class="anchored" data-anchor-id="transformations-and-scaling-of-space">Transformations and Scaling of Space</h4>
<p>Consider a linear transformation <span class="math inline">\(T: \mathbb{R}^n \to \mathbb{R}^n\)</span> represented by a square matrix <span class="math inline">\(A\)</span>. When <span class="math inline">\(A\)</span> acts on vectors, it reshapes space: it stretches, compresses, rotates, reflects, or shears regions.</p>
<ul>
<li>If you apply <span class="math inline">\(A\)</span> to a unit square in <span class="math inline">\(\mathbb{R}^2\)</span>, the image is a parallelogram.</li>
<li>If you apply <span class="math inline">\(A\)</span> to a unit cube in <span class="math inline">\(\mathbb{R}^3\)</span>, the image is a parallelepiped.</li>
<li>In general, the determinant of <span class="math inline">\(A\)</span> tells us how the measure (area, volume, hyper-volume) of the shape has changed.</li>
</ul>
</section>
<section id="determinant-as-signed-scale-factor" class="level4">
<h4 class="anchored" data-anchor-id="determinant-as-signed-scale-factor">Determinant as Signed Scale Factor</h4>
<ul>
<li><span class="math inline">\(|\det(A)|\)</span> = the scale factor for areas (2D), volumes (3D), or n-dimensional content.</li>
<li>If <span class="math inline">\(\det(A) = 0\)</span>, the transformation collapses space into a lower dimension, flattening all volume away.</li>
<li>If <span class="math inline">\(\det(A) &gt; 0\)</span>, the orientation of space is preserved.</li>
<li>If <span class="math inline">\(\det(A) &lt; 0\)</span>, the orientation is flipped (like a reflection in a mirror).</li>
</ul>
<p>Thus, determinants are not just numbers-they carry both magnitude and sign, telling us about size and handedness.</p>
</section>
<section id="d-case-area-of-parallelogram" class="level4">
<h4 class="anchored" data-anchor-id="d-case-area-of-parallelogram">2D Case: Area of Parallelogram</h4>
<p>Take two column vectors <span class="math inline">\(u,v \in \mathbb{R}^2\)</span>. Place them as columns in a matrix:</p>
<p><span class="math display">\[
A = \begin{bmatrix} u &amp; v \end{bmatrix}.
\]</span></p>
<p>The absolute value of the determinant gives the area of the parallelogram spanned by <span class="math inline">\(u\)</span> and <span class="math inline">\(v\)</span>:</p>
<p><span class="math display">\[
\text{Area} = |\det(A)|.
\]</span></p>
<p>Example:</p>
<p><span class="math display">\[
A = \begin{bmatrix} 2 &amp; 1 \\ 1 &amp; 3 \end{bmatrix}.
\]</span></p>
<p>Then <span class="math inline">\(\det(A) = (2)(3) - (1)(1) = 5\)</span>. The unit square maps to a parallelogram of area 5.</p>
</section>
<section id="d-case-volume-of-parallelepiped" class="level4">
<h4 class="anchored" data-anchor-id="d-case-volume-of-parallelepiped">3D Case: Volume of Parallelepiped</h4>
<p>For three vectors <span class="math inline">\(u,v,w \in \mathbb{R}^3\)</span>, form a matrix</p>
<p><span class="math display">\[
A = \begin{bmatrix} u &amp; v &amp; w \end{bmatrix}.
\]</span></p>
<p>Then the absolute determinant gives the volume of the parallelepiped:</p>
<p><span class="math display">\[
\text{Volume} = |\det(A)|.
\]</span></p>
<p>Geometrically, this is the scalar triple product:</p>
<p><span class="math display">\[
\det(A) = u \cdot (v \times w).
\]</span></p>
<p>Example:</p>
<p><span class="math display">\[
A = \begin{bmatrix} 1 &amp; 0 &amp; 0 \\ 0 &amp; 2 &amp; 0 \\ 0 &amp; 0 &amp; 3 \end{bmatrix}, \quad \det(A) = 6.
\]</span></p>
<p>So the unit cube is stretched into a box with volume 6.</p>
</section>
<section id="orientation-and-signed-measure" class="level4">
<h4 class="anchored" data-anchor-id="orientation-and-signed-measure">Orientation and Signed Measure</h4>
<p>Determinants do more than measure size-they also detect orientation:</p>
<ul>
<li>In 2D, flipping x and y axes changes the sign of the determinant.</li>
<li>In 3D, swapping two vectors changes the “handedness” (right-hand rule becomes left-hand rule).</li>
</ul>
<p>This explains why determinants can be negative: they mark transformations that reverse orientation.</p>
</section>
<section id="higher-dimensions" class="level4">
<h4 class="anchored" data-anchor-id="higher-dimensions">Higher Dimensions</h4>
<p>In <span class="math inline">\(\mathbb{R}^n\)</span>, determinants extend the same idea. A unit hypercube (side length 1) is transformed into an n-dimensional parallelotope, whose volume is given by <span class="math inline">\(|\det(A)|\)</span>.</p>
<p>Though we cannot visualize beyond 3D, the concept generalizes smoothly: determinants encode how much an n-dimensional object is stretched or collapsed.</p>
</section>
<section id="applications-13" class="level4">
<h4 class="anchored" data-anchor-id="applications-13">Applications</h4>
<ol type="1">
<li>Geometry: Computing areas, volumes, and orientation directly from vectors.</li>
<li>Computer Graphics: Determinants detect whether a transformation preserves or flips orientation, useful in rendering.</li>
<li>Physics: Determinants describe Jacobians for coordinate changes in integrals, adjusting volume elements.</li>
<li>Engineering: Determinants quantify deformation and stress in materials (strain tensors).</li>
<li>Data Science: Determinants of covariance matrices encode “volume” of uncertainty ellipsoids.</li>
</ol>
</section>
<section id="why-it-matters-47" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-47">Why It Matters</h4>
<ol type="1">
<li>Determinants connect algebra (formulas) to geometry (shapes).</li>
<li>They explain why some transformations lose information: <span class="math inline">\(\det=0\)</span>.</li>
<li>They preserve orientation, key for consistent physical laws and geometry.</li>
<li>They prepare us for advanced tools like Jacobians, eigenvalues, and volume-preserving maps.</li>
</ol>
</section>
<section id="try-it-yourself-50" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-50">Try It Yourself</h4>
<ol type="1">
<li>Compute the area of the parallelogram spanned by <span class="math inline">\((1,2)\)</span> and <span class="math inline">\((3,1)\)</span>.</li>
<li>Find the volume of the parallelepiped defined by vectors <span class="math inline">\((1,0,0),(0,1,0),(1,1,1)\)</span>.</li>
<li>Show that swapping two columns of a matrix flips the sign of the determinant but keeps absolute value unchanged.</li>
<li>Challenge: Explain why <span class="math inline">\(\det(A)\)</span> gives the scaling factor for integrals under change of variables.</li>
</ol>
<p>Determinants begin as algebraic formulas, but their real meaning lies in geometry: they measure how linear transformations scale, compress, or flip space itself.</p>
</section>
</section>
<section id="determinant-via-linear-rules" class="level3">
<h3 class="anchored" data-anchor-id="determinant-via-linear-rules">52. Determinant via Linear Rules</h3>
<p>The determinant is not just a mysterious formula-it is a function built from a few simple rules that uniquely determine its behavior. These rules, often called determinant axioms, allow us to see the determinant as the only measure of “signed volume” compatible with linear algebra. Understanding these rules gives clarity: instead of memorizing expansion formulas, we see why determinants behave as they do.</p>
<section id="the-setup" class="level4">
<h4 class="anchored" data-anchor-id="the-setup">The Setup</h4>
<p>Take a square matrix <span class="math inline">\(A \in \mathbb{R}^{n \times n}\)</span>. Think of <span class="math inline">\(A\)</span> as a list of <span class="math inline">\(n\)</span> column vectors:</p>
<p><span class="math display">\[
A = \begin{bmatrix} a_1 &amp; a_2 &amp; \cdots &amp; a_n \end{bmatrix}.
\]</span></p>
<p>The determinant is a function <span class="math inline">\(\det: \mathbb{R}^{n \times n} \to \mathbb{R}\)</span> that assigns a single number to <span class="math inline">\(A\)</span>. Geometrically, it gives the signed volume of the parallelotope spanned by <span class="math inline">\((a_1, \dots, a_n)\)</span>. Algebraically, it follows three key rules.</p>
</section>
<section id="rule-1-linearity-in-each-column" class="level4">
<h4 class="anchored" data-anchor-id="rule-1-linearity-in-each-column">Rule 1: Linearity in Each Column</h4>
<p>If you scale one column by a scalar <span class="math inline">\(c\)</span>, the determinant scales by <span class="math inline">\(c\)</span>.</p>
<p><span class="math display">\[
\det(a_1, \dots, c a_j, \dots, a_n) = c \cdot \det(a_1, \dots, a_j, \dots, a_n).
\]</span></p>
<p>If you replace a column with a sum, the determinant splits:</p>
<p><span class="math display">\[
\det(a_1, \dots, (b+c), \dots, a_n) = \det(a_1, \dots, b, \dots, a_n) + \det(a_1, \dots, c, \dots, a_n).
\]</span></p>
<p>This linearity means determinants behave predictably with respect to scaling and addition.</p>
</section>
<section id="rule-2-alternating-property" class="level4">
<h4 class="anchored" data-anchor-id="rule-2-alternating-property">Rule 2: Alternating Property</h4>
<p>If two columns are the same, the determinant is zero:</p>
<p><span class="math display">\[
\det(\dots, a_i, \dots, a_i, \dots) = 0.
\]</span></p>
<p>This makes sense geometrically: if two spanning vectors are identical, they collapse the volume to zero.</p>
<p>Equivalently: if you swap two columns, the determinant flips sign:</p>
<p><span class="math display">\[
\det(\dots, a_i, \dots, a_j, \dots) = -\det(\dots, a_j, \dots, a_i, \dots).
\]</span></p>
</section>
<section id="rule-3-normalization" class="level4">
<h4 class="anchored" data-anchor-id="rule-3-normalization">Rule 3: Normalization</h4>
<p>The determinant of the identity matrix is 1:</p>
<p><span class="math display">\[
\det(I_n) = 1.
\]</span></p>
<p>This anchors the function: the unit cube has volume 1, with positive orientation.</p>
</section>
<section id="consequence-uniqueness" class="level4">
<h4 class="anchored" data-anchor-id="consequence-uniqueness">Consequence: Uniqueness</h4>
<p>These three rules (linearity, alternating, normalization) uniquely define the determinant. Any function satisfying them must be the determinant. This makes it less of an arbitrary formula and more of a natural consequence of linear structure.</p>
</section>
<section id="small-cases-explicit-formulas" class="level4">
<h4 class="anchored" data-anchor-id="small-cases-explicit-formulas">Small Cases: Explicit Formulas</h4>
<ul>
<li><p>2×2 matrices:</p>
<p><span class="math display">\[
\det \begin{bmatrix} a &amp; b \\ c &amp; d \end{bmatrix} = ad - bc.
\]</span></p>
<p>This formula arises directly from the rules: linearity in columns and alternating sign when swapping them.</p></li>
<li><p>3×3 matrices: Expansion formula:</p>
<p><span class="math display">\[
\det \begin{bmatrix}
a &amp; b &amp; c \\
d &amp; e &amp; f \\
g &amp; h &amp; i
\end{bmatrix}
= aei + bfg + cdh - ceg - bdi - afh.
\]</span></p></li>
</ul>
<p>This looks complicated, but it comes from systematically applying the rules to break down the volume.</p>
</section>
<section id="geometric-interpretation-of-the-rules" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-of-the-rules">Geometric Interpretation of the Rules</h4>
<ol type="1">
<li>Linearity: Stretching one side of a parallelogram or parallelepiped scales the area or volume.</li>
<li>Alternating: If two sides collapse into the same direction, the area/volume vanishes. Swapping sides flips orientation.</li>
<li>Normalization: The unit cube has size 1 by definition.</li>
</ol>
<p>Together, these mirror geometric intuition exactly.</p>
</section>
<section id="higher-dimensional-generalization" class="level4">
<h4 class="anchored" data-anchor-id="higher-dimensional-generalization">Higher-Dimensional Generalization</h4>
<p>In <span class="math inline">\(\mathbb{R}^n\)</span>, determinants measure oriented hyper-volume. For example, in 4D, determinants give the “4-volume” of a parallelotope. Though impossible to picture, the same rules apply.</p>
</section>
<section id="applications-14" class="level4">
<h4 class="anchored" data-anchor-id="applications-14">Applications</h4>
<ol type="1">
<li>Defining area and volume: Determinants provide a universal formula for computing geometric sizes from coordinates.</li>
<li>Jacobian determinants: Used in calculus when changing variables in multiple integrals.</li>
<li>Orientation detection: Whether transformations preserve handedness in geometry or physics.</li>
<li>Computer graphics: Ensuring consistent orientation of polygons and meshes.</li>
</ol>
</section>
<section id="why-it-matters-48" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-48">Why It Matters</h4>
<p>Determinants are not arbitrary. They arise naturally once we demand a function that is linear in columns, alternating, and normalized. This explains why so many different formulas and properties agree: they are all shadows of the same underlying definition.</p>
</section>
<section id="try-it-yourself-51" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-51">Try It Yourself</h4>
<ol type="1">
<li>Show that scaling one column by 3 multiplies the determinant by 3.</li>
<li>Compute the determinant of <span class="math inline">\(\begin{bmatrix} 1 &amp; 2 \\ 2 &amp; 4 \end{bmatrix}\)</span> and explain why it is zero.</li>
<li>Swap two columns in <span class="math inline">\(\begin{bmatrix} 1 &amp; 2 \\ 3 &amp; 4 \end{bmatrix}\)</span> and confirm the determinant changes sign.</li>
<li>Challenge: Use only the three rules to derive the <span class="math inline">\(2 \times 2\)</span> determinant formula.</li>
</ol>
<p>The determinant is the unique bridge between algebra and geometry, born from a handful of simple but powerful rules.</p>
</section>
</section>
<section id="determinant-and-row-operations" class="level3">
<h3 class="anchored" data-anchor-id="determinant-and-row-operations">53. Determinant and Row Operations</h3>
<p>One of the most practical ways to compute determinants is by using row operations, the same tools used in Gaussian elimination. Determinants interact with these operations in very structured ways. By understanding the rules, we can compute determinants systematically without resorting to long expansion formulas.</p>
<section id="row-operations-recap" class="level4">
<h4 class="anchored" data-anchor-id="row-operations-recap">Row Operations Recap</h4>
<p>There are three elementary row operations:</p>
<ol type="1">
<li>Row Swap (R<span class="math inline">\(_i \leftrightarrow\)</span> R<span class="math inline">\(_j\)</span>) – exchange two rows.</li>
<li>Row Scaling (c·R<span class="math inline">\(_i\)</span>) – multiply a row by a scalar <span class="math inline">\(c\)</span>.</li>
<li>Row Replacement (R<span class="math inline">\(_i\)</span> + c·R<span class="math inline">\(_j\)</span>) – replace one row with itself plus a multiple of another row.</li>
</ol>
<p>Since the determinant is defined in terms of linearity and alternation of rows (or columns), each operation has a clear effect.</p>
</section>
<section id="rule-1-row-swap-changes-sign" class="level4">
<h4 class="anchored" data-anchor-id="rule-1-row-swap-changes-sign">Rule 1: Row Swap Changes Sign</h4>
<p>If you swap two rows, the determinant changes sign:</p>
<p><span class="math display">\[
\det(A \text{ with } R_i \leftrightarrow R_j) = -\det(A).
\]</span></p>
<p>Reason: Swapping two spanning vectors flips orientation. In 2D, swapping basis vectors flips a parallelogram across the diagonal, reversing handedness.</p>
</section>
<section id="rule-2-row-scaling-multiplies-determinant" class="level4">
<h4 class="anchored" data-anchor-id="rule-2-row-scaling-multiplies-determinant">Rule 2: Row Scaling Multiplies Determinant</h4>
<p>If you multiply a row by a scalar <span class="math inline">\(c\)</span>:</p>
<p><span class="math display">\[
\det(A \text{ with } cR_i) = c \cdot \det(A).
\]</span></p>
<p>Reason: Scaling one side of a parallelogram multiplies its area; scaling one dimension of a cube multiplies its volume.</p>
</section>
<section id="rule-3-row-replacement-leaves-determinant-unchanged" class="level4">
<h4 class="anchored" data-anchor-id="rule-3-row-replacement-leaves-determinant-unchanged">Rule 3: Row Replacement Leaves Determinant Unchanged</h4>
<p>If you replace one row with itself plus a multiple of another row:</p>
<p><span class="math display">\[
\det(A \text{ with } R_i \to R_i + cR_j) = \det(A).
\]</span></p>
<p>Reason: Adding a multiple of one spanning vector to another doesn’t change the spanned volume. The parallelogram or parallelepiped is sheared, but its area or volume remains the same.</p>
</section>
<section id="why-these-rules-work-together" class="level4">
<h4 class="anchored" data-anchor-id="why-these-rules-work-together">Why These Rules Work Together</h4>
<p>These three rules align perfectly with the determinant axioms:</p>
<ul>
<li>Alternating → row swaps flip sign.</li>
<li>Linearity → scaling multiplies by scalar.</li>
<li>Normalization → row replacement preserves measure.</li>
</ul>
<p>Thus, row operations provide a complete framework for computing determinants.</p>
</section>
<section id="computing-determinants-with-elimination" class="level4">
<h4 class="anchored" data-anchor-id="computing-determinants-with-elimination">Computing Determinants with Elimination</h4>
<p>To compute <span class="math inline">\(\det(A)\)</span>:</p>
<ol type="1">
<li>Perform Gaussian elimination to reduce <span class="math inline">\(A\)</span> to an upper triangular matrix <span class="math inline">\(U\)</span>.</li>
<li>Track how row swaps and scalings affect the determinant.</li>
<li>Use the fact that the determinant of a triangular matrix is the product of its diagonal entries.</li>
</ol>
<p>Example:</p>
<p><span class="math display">\[
A = \begin{bmatrix} 2 &amp; 1 &amp; 3 \\ 4 &amp; 1 &amp; 7 \\ -2 &amp; 5 &amp; 1 \end{bmatrix}.
\]</span></p>
<ul>
<li><p>Step 1: <span class="math inline">\(R_2 \to R_2 - 2R_1\)</span>, <span class="math inline">\(R_3 \to R_3 + R_1\)</span>. No determinant change.</p></li>
<li><p>Step 2: Upper triangular form emerges:</p>
<p><span class="math display">\[
U = \begin{bmatrix} 2 &amp; 1 &amp; 3 \\ 0 &amp; -1 &amp; 1 \\ 0 &amp; 0 &amp; -5 \end{bmatrix}.
\]</span></p></li>
<li><p>Step 3: Determinant is product of diagonals: <span class="math inline">\(\det(A) = 2 \cdot (-1) \cdot (-5) = 10.\)</span></p></li>
</ul>
<p>Efficient, clear, and no messy cofactor expansions.</p>
</section>
<section id="geometric-view" class="level4">
<h4 class="anchored" data-anchor-id="geometric-view">Geometric View</h4>
<ul>
<li>Row swap: Flips orientation of the volume.</li>
<li>Row scaling: Stretches or compresses one dimension of the volume.</li>
<li>Row replacement: Slides faces of the volume without changing its size.</li>
</ul>
<p>This geometric reasoning reinforces why the rules are natural.</p>
</section>
<section id="applications-15" class="level4">
<h4 class="anchored" data-anchor-id="applications-15">Applications</h4>
<ol type="1">
<li>Efficient computation: Algorithms for large determinants (LU decomposition) are based on row operations.</li>
<li>Numerical analysis: Determinant rules help detect stability and singularity.</li>
<li>Geometry: Orientation tests for polygons rely on row swap rules.</li>
<li>Theoretical results: Many determinant identities are derived directly from row operation behavior.</li>
</ol>
</section>
<section id="why-it-matters-49" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-49">Why It Matters</h4>
<ul>
<li>Determinants link algebra to geometry, but computation requires efficient methods.</li>
<li>Row operations give a hands-on toolkit: they’re the backbone of practical determinant computation.</li>
<li>Understanding these rules explains why algorithms like LU factorization work so well.</li>
</ul>
</section>
<section id="try-it-yourself-52" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-52">Try It Yourself</h4>
<ol type="1">
<li>Compute the determinant of <span class="math inline">\(\begin{bmatrix} 1 &amp; 2 &amp; 3 \\ 4 &amp; 5 &amp; 6 \\ 7 &amp; 8 &amp; 9 \end{bmatrix}\)</span> using elimination.</li>
<li>Verify that replacing <span class="math inline">\(R_2 \to R_2 + 3R_1\)</span> does not change the determinant.</li>
<li>Check how many sign flips occur if you reorder rows into strictly increasing order.</li>
<li>Challenge: Prove that elimination combined with these rules always leads to the triangular product formula.</li>
</ol>
<p>Determinants are not meant to be expanded by brute force; row operations transform the problem into a clear sequence of steps, connecting algebraic efficiency with geometric intuition.</p>
</section>
</section>
<section id="triangular-matrices-and-product-of-diagonals" class="level3">
<h3 class="anchored" data-anchor-id="triangular-matrices-and-product-of-diagonals">54. Triangular Matrices and Product of Diagonals</h3>
<p>Among all types of matrices, triangular matrices stand out for their simplicity. These are matrices where every entry either above or below the main diagonal is zero. What makes them especially important is that their determinants can be computed almost instantly: the determinant of a triangular matrix is simply the product of its diagonal entries. This property is not only computationally convenient, it also reveals deep connections between determinants, row operations, and structure in linear algebra.</p>
<section id="triangular-matrices-defined" class="level4">
<h4 class="anchored" data-anchor-id="triangular-matrices-defined">Triangular Matrices Defined</h4>
<p>A square matrix is called upper triangular if all entries below the main diagonal are zero, and lower triangular if all entries above the diagonal are zero.</p>
<ul>
<li><p>Upper triangular example:</p>
<p><span class="math display">\[
U = \begin{bmatrix}
2 &amp; 5 &amp; -1 \\
0 &amp; 3 &amp; 4 \\
0 &amp; 0 &amp; 7
\end{bmatrix}.
\]</span></p></li>
<li><p>Lower triangular example:</p>
<p><span class="math display">\[
L = \begin{bmatrix}
4 &amp; 0 &amp; 0 \\
-2 &amp; 5 &amp; 0 \\
1 &amp; 3 &amp; 6
\end{bmatrix}.
\]</span></p></li>
</ul>
<p>Both share the key feature: “everything off one side of the diagonal vanishes.”</p>
</section>
<section id="determinant-rule" class="level4">
<h4 class="anchored" data-anchor-id="determinant-rule">Determinant Rule</h4>
<p>For any triangular matrix,</p>
<p><span class="math display">\[
\det(T) = \prod_{i=1}^n t_{ii},
\]</span></p>
<p>where <span class="math inline">\(t_{ii}\)</span> are the diagonal entries.</p>
<p>So for the upper triangular <span class="math inline">\(U\)</span> above,</p>
<p><span class="math display">\[
\det(U) = 2 \times 3 \times 7 = 42.
\]</span></p>
</section>
<section id="why-this-works" class="level4">
<h4 class="anchored" data-anchor-id="why-this-works">Why This Works</h4>
<p>The determinant is multilinear and alternating. When you expand it (e.g., via cofactor expansion), only one product of entries survives in the expansion: the one that picks exactly the diagonal terms.</p>
<ul>
<li>If you try to pick an off-diagonal entry in a row, you eventually get stuck with a zero entry because of the triangular shape.</li>
<li>The only surviving term is the product of the diagonals, with sign <span class="math inline">\(+1\)</span>.</li>
</ul>
<p>This elegant reasoning explains why the rule holds universally.</p>
</section>
<section id="connection-to-row-operations" class="level4">
<h4 class="anchored" data-anchor-id="connection-to-row-operations">Connection to Row Operations</h4>
<p>Recall: elimination reduces any square matrix to an upper triangular form. Once triangular, the determinant is simply the product of the diagonals, adjusted for row swaps and scalings.</p>
<p>Thus, triangular matrices are not just simple-they are the end goal of elimination algorithms for determinant computation.</p>
</section>
<section id="geometric-meaning-8" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-8">Geometric Meaning</h4>
<p>In geometric terms:</p>
<ul>
<li>A triangular matrix represents a transformation where each coordinate direction depends only on itself and earlier coordinates.</li>
<li>The determinant equals the product of scaling along each axis.</li>
<li>Example: In 3D, scaling x by 2, y by 3, and z by 7 gives a volume scaling of <span class="math inline">\(2 \cdot 3 \cdot 7 = 42\)</span>.</li>
</ul>
<p>Even if shear is present in the upper entries, the determinant ignores it-it only cares about the pure diagonal scaling.</p>
</section>
<section id="applications-16" class="level4">
<h4 class="anchored" data-anchor-id="applications-16">Applications</h4>
<ol type="1">
<li>Efficient computation: LU decomposition reduces determinants to diagonal product form.</li>
<li>Theoretical proofs: Many determinant identities reduce to triangular cases.</li>
<li>Numerical stability: Triangular matrices are well-behaved in computation, crucial for algorithms in numerical linear algebra.</li>
<li>Eigenvalues: For triangular matrices, eigenvalues are exactly the diagonal entries; thus determinant = product of eigenvalues.</li>
<li>Computer graphics: Triangular forms simplify geometric transformations.</li>
</ol>
</section>
<section id="why-it-matters-50" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-50">Why It Matters</h4>
<ol type="1">
<li>Provides the fastest way to compute determinants in special cases.</li>
<li>Serves as the computational foundation for general determinant algorithms.</li>
<li>Connects determinants directly to eigenvalues and scaling factors.</li>
<li>Illustrates how elimination transforms complexity into simplicity.</li>
</ol>
</section>
<section id="try-it-yourself-53" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-53">Try It Yourself</h4>
<ol type="1">
<li><p>Compute the determinant of</p>
<p><span class="math display">\[
\begin{bmatrix}
1 &amp; 2 &amp; 3 \\
0 &amp; 4 &amp; 5 \\
0 &amp; 0 &amp; 6
\end{bmatrix}.
\]</span></p>
<p>(Check: it should equal <span class="math inline">\(1 \cdot 4 \cdot 6\)</span>).</p></li>
<li><p>Verify that a lower triangular matrix with diagonal entries <span class="math inline">\((2, -1, 5)\)</span> has determinant <span class="math inline">\(-10\)</span>.</p></li>
<li><p>Explain why an upper triangular matrix with a zero on the diagonal must have determinant 0.</p></li>
<li><p>Challenge: Prove that every square matrix can be reduced to triangular form with determinant tracked by elimination steps.</p></li>
</ol>
<p>The triangular case reveals the heart of determinants: a product of diagonal scalings, stripped of all extra noise. It is the simplest lens through which determinants become transparent.</p>
</section>
</section>
<section id="the-multiplicative-property-of-determinants-detab-detadetb" class="level3">
<h3 class="anchored" data-anchor-id="the-multiplicative-property-of-determinants-detab-detadetb">55. The Multiplicative Property of Determinants: <span class="math inline">\(\det(AB) = \det(A)\det(B)\)</span></h3>
<p>One of the most remarkable and useful facts about determinants is that they multiply across matrix products. For two square matrices of the same size,</p>
<p><span class="math display">\[
\det(AB) = \det(A) \cdot \det(B).
\]</span></p>
<p>This property is fundamental: it connects algebra (matrix multiplication) with geometry (scaling volumes) and is essential for proofs, computations, and applications across mathematics, physics, and engineering.</p>
<section id="the-statement-in-words" class="level4">
<h4 class="anchored" data-anchor-id="the-statement-in-words">The Statement in Words</h4>
<ul>
<li>If you first apply a linear transformation <span class="math inline">\(B\)</span>, and then apply <span class="math inline">\(A\)</span>, the total scaling of space is the product of their individual scalings.</li>
<li>Determinants track exactly this: the signed volume change under linear transformations.</li>
</ul>
</section>
<section id="geometric-intuition-4" class="level4">
<h4 class="anchored" data-anchor-id="geometric-intuition-4">Geometric Intuition</h4>
<p>Think of <span class="math inline">\(\det(A)\)</span> as the signed scale factor by which <span class="math inline">\(A\)</span> changes volume.</p>
<ol type="1">
<li>Apply <span class="math inline">\(B\)</span>: a unit cube becomes some parallelepiped with volume <span class="math inline">\(|\det(B)|\)</span>.</li>
<li>Apply <span class="math inline">\(A\)</span>: the new parallelepiped scales again by <span class="math inline">\(|\det(A)|\)</span>.</li>
<li>Total effect: volume scales by <span class="math inline">\(|\det(A)| \times |\det(B)|\)</span>.</li>
</ol>
<p>The orientation flips are also consistent: if both flip (negative determinants), the total orientation is preserved (positive product).</p>
</section>
<section id="algebraic-reasoning" class="level4">
<h4 class="anchored" data-anchor-id="algebraic-reasoning">Algebraic Reasoning</h4>
<p>The proof can be approached in multiple ways:</p>
<ol type="1">
<li><p>Row Operations and Elimination:</p>
<ul>
<li><span class="math inline">\(A\)</span> and <span class="math inline">\(B\)</span> can be factored into elementary matrices (row swaps, scalings, replacements).</li>
<li>Determinants behave predictably for each operation.</li>
<li>Since both sides agree for elementary operations and determinant is multiplicative, the identity holds in general.</li>
</ul></li>
<li><p>Abstract Characterization:</p>
<ul>
<li>Determinants are the unique multilinear alternating functions normalized at the identity.</li>
<li>Composition of linear maps preserves this property, so multiplicativity follows.</li>
</ul></li>
</ol>
</section>
<section id="small-cases" class="level4">
<h4 class="anchored" data-anchor-id="small-cases">Small Cases</h4>
<ul>
<li><p>2×2 matrices:</p>
<p><span class="math display">\[
A = \begin{bmatrix} a &amp; b \\ c &amp; d \end{bmatrix}, \quad
B = \begin{bmatrix} e &amp; f \\ g &amp; h \end{bmatrix}.
\]</span></p>
<p>Compute <span class="math inline">\(AB\)</span>, then <span class="math inline">\(\det(AB)\)</span>. After expansion, you find: <span class="math inline">\(\det(AB) = (ad - bc)(eh - fg) = \det(A)\det(B).\)</span></p></li>
<li><p>3×3 matrices: A direct computation is messy, but the property still holds and is consistent with elimination proofs.</p></li>
</ul>
</section>
<section id="key-consequences" class="level4">
<h4 class="anchored" data-anchor-id="key-consequences">Key Consequences</h4>
<ol type="1">
<li><p>Determinant of a Power:</p>
<p><span class="math display">\[
\det(A^k) = (\det(A))^k.
\]</span></p>
<p>Geometric meaning: applying the same transformation <span class="math inline">\(k\)</span> times multiplies volume scale repeatedly.</p></li>
<li><p>Inverse Matrix: If <span class="math inline">\(A\)</span> is invertible,</p>
<p><span class="math display">\[
\det(A^{-1}) = \frac{1}{\det(A)}.
\]</span></p></li>
<li><p>Eigenvalues: Since <span class="math inline">\(\det(A)\)</span> is the product of eigenvalues, this property matches the fact that eigenvalues multiply under matrix multiplication (when considered via characteristic polynomials).</p></li>
</ol>
</section>
<section id="geometric-meaning-in-higher-dimensions" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-in-higher-dimensions">Geometric Meaning in Higher Dimensions</h4>
<ul>
<li>If <span class="math inline">\(B\)</span> scales space by 3 and flips it (det = –3), and <span class="math inline">\(A\)</span> scales by 2 without flipping (det = 2), then <span class="math inline">\(AB\)</span> scales by –6, consistent with the rule.</li>
<li>Determinants encapsulate both magnitude (volume scaling) and sign (orientation). Multiplicativity ensures these combine correctly.</li>
</ul>
</section>
<section id="applications-17" class="level4">
<h4 class="anchored" data-anchor-id="applications-17">Applications</h4>
<ol type="1">
<li>Change of Variables in Calculus: The Jacobian determinant follows this multiplicative rule, ensuring transformations compose consistently.</li>
<li>Group Theory: <span class="math inline">\(\det\)</span> defines a group homomorphism from the general linear group <span class="math inline">\(GL_n\)</span> to the nonzero reals under multiplication.</li>
<li>Numerical Analysis: Determinant multiplicativity underlies LU decomposition methods.</li>
<li>Physics: In mechanics and relativity, volume elements transform consistently under successive transformations.</li>
<li>Cryptography and Coding Theory: Determinants in modular arithmetic rely on this multiplicative property to preserve structure.</li>
</ol>
</section>
<section id="why-it-matters-51" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-51">Why It Matters</h4>
<ul>
<li>Guarantees consistency: determinants match our intuition about scaling.</li>
<li>Simplifies computation: determinants of factorizations can be obtained by multiplying smaller pieces.</li>
<li>Provides theoretical structure: <span class="math inline">\(\det\)</span> is a homomorphism, embedding linear algebra into the algebra of scalars.</li>
</ul>
</section>
<section id="try-it-yourself-54" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-54">Try It Yourself</h4>
<ol type="1">
<li><p>Verify <span class="math inline">\(\det(AB) = \det(A)\det(B)\)</span> for</p>
<p><span class="math display">\[
A = \begin{bmatrix} 2 &amp; 1 \\ 0 &amp; 3 \end{bmatrix}, \quad
B = \begin{bmatrix} 1 &amp; 4 \\ 0 &amp; -2 \end{bmatrix}.
\]</span></p></li>
<li><p>Prove that <span class="math inline">\(\det(A^{-1}) = 1/\det(A)\)</span> using the multiplicative rule.</p></li>
<li><p>Show that if <span class="math inline">\(\det(A)=0\)</span>, then <span class="math inline">\(\det(AB)=0\)</span> for any <span class="math inline">\(B\)</span>. Explain why this makes sense geometrically.</p></li>
<li><p>Challenge: Using row operations, show explicitly how multiplicativity emerges from properties of elementary matrices.</p></li>
</ol>
<p>The rule <span class="math inline">\(\det(AB) = \det(A)\det(B)\)</span> transforms determinants from a mysterious calculation into a natural and consistent measure of how linear transformations combine.</p>
</section>
</section>
<section id="invertibility-and-zero-determinant" class="level3">
<h3 class="anchored" data-anchor-id="invertibility-and-zero-determinant">56. Invertibility and Zero Determinant</h3>
<p>The determinant is more than a geometric scale factor-it is the ultimate test of whether a matrix is invertible. A square matrix <span class="math inline">\(A \in \mathbb{R}^{n \times n}\)</span> has an inverse if and only if its determinant is nonzero. When the determinant vanishes, the matrix collapses space into a lower dimension, losing information that no transformation can undo.</p>
<section id="the-criterion" class="level4">
<h4 class="anchored" data-anchor-id="the-criterion">The Criterion</h4>
<p><span class="math display">\[
A \text{ invertible } \iff \det(A) \neq 0.
\]</span></p>
<ul>
<li>If <span class="math inline">\(\det(A) \neq 0\)</span>, the transformation stretches or shrinks space but never flattens it. Every output corresponds to exactly one input, so <span class="math inline">\(A^{-1}\)</span> exists.</li>
<li>If <span class="math inline">\(\det(A) = 0\)</span>, some directions are squashed into lower dimensions. Information is destroyed, so no inverse exists.</li>
</ul>
</section>
<section id="geometric-meaning-9" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-9">Geometric Meaning</h4>
<ol type="1">
<li><p>In 2D:</p>
<ul>
<li>A nonzero determinant means the unit square is sent to a parallelogram with nonzero area.</li>
<li>A zero determinant means the square collapses into a line segment or a point.</li>
</ul></li>
<li><p>In 3D:</p>
<ul>
<li>Nonzero determinant → unit cube becomes a 3D parallelepiped with volume.</li>
<li>Zero determinant → cube flattens into a sheet or a line; 3D volume is lost.</li>
</ul></li>
<li><p>In Higher Dimensions:</p>
<ul>
<li>Nonzero determinant preserves n-dimensional volume.</li>
<li>Zero determinant collapses dimension, destroying invertibility.</li>
</ul></li>
</ol>
</section>
<section id="algebraic-meaning" class="level4">
<h4 class="anchored" data-anchor-id="algebraic-meaning">Algebraic Meaning</h4>
<ul>
<li><p>The determinant is the product of eigenvalues:</p>
<p><span class="math display">\[
\det(A) = \lambda_1 \lambda_2 \cdots \lambda_n.
\]</span></p>
<p>If any eigenvalue is zero, then <span class="math inline">\(\det(A) = 0\)</span> and the matrix is singular (not invertible).</p></li>
<li><p>Equivalently, a zero determinant means the matrix has linearly dependent columns or rows. This dependence implies redundancy: not all directions are independent, so the mapping cannot be one-to-one.</p></li>
</ul>
</section>
<section id="connection-with-linear-systems" class="level4">
<h4 class="anchored" data-anchor-id="connection-with-linear-systems">Connection with Linear Systems</h4>
<ul>
<li><p>If <span class="math inline">\(\det(A) \neq 0\)</span>:</p>
<ul>
<li>The system <span class="math inline">\(Ax = b\)</span> has a unique solution for every <span class="math inline">\(b\)</span>.</li>
<li>The inverse matrix <span class="math inline">\(A^{-1}\)</span> exists and satisfies <span class="math inline">\(x = A^{-1}b\)</span>.</li>
</ul></li>
<li><p>If <span class="math inline">\(\det(A) = 0\)</span>:</p>
<ul>
<li>Either no solutions (inconsistent system) or infinitely many solutions (dependent equations).</li>
<li>The mapping <span class="math inline">\(x \mapsto Ax\)</span> cannot be reversed.</li>
</ul></li>
</ul>
</section>
<section id="example-invertible-vs.-singular" class="level4">
<h4 class="anchored" data-anchor-id="example-invertible-vs.-singular">Example: Invertible vs.&nbsp;Singular</h4>
<ol type="1">
<li></li>
</ol>
<p><span class="math display">\[
A = \begin{bmatrix} 2 &amp; 1 \\ 1 &amp; 3 \end{bmatrix}, \quad \det(A) = 5 \neq 0.
\]</span></p>
<p>Invertible.</p>
<ol start="2" type="1">
<li></li>
</ol>
<p><span class="math display">\[
B = \begin{bmatrix} 2 &amp; 4 \\ 1 &amp; 2 \end{bmatrix}, \quad \det(B) = 0.
\]</span></p>
<p>Not invertible, since the second column is just twice the first.</p>
</section>
<section id="applications-18" class="level4">
<h4 class="anchored" data-anchor-id="applications-18">Applications</h4>
<ol type="1">
<li>Solving Systems: Inverse-based methods rely on nonzero determinants.</li>
<li>Numerical Methods: Detecting near-singularity warns of unstable solutions.</li>
<li>Geometry: A singular matrix corresponds to degenerate shapes (flattened, collapsed).</li>
<li>Physics: In mechanics and relativity, invertibility ensures that transformations can be reversed.</li>
<li>Computer Graphics: Non-invertible transformations crush dimensions, breaking rendering pipelines.</li>
</ol>
</section>
<section id="why-it-matters-52" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-52">Why It Matters</h4>
<ul>
<li>Determinants provide a single scalar test for invertibility.</li>
<li>This connects geometry (volume collapse), algebra (linear dependence), and analysis (solvability of systems).</li>
<li>The zero/nonzero divide is one of the sharpest and most important in all of linear algebra.</li>
</ul>
</section>
<section id="try-it-yourself-55" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-55">Try It Yourself</h4>
<ol type="1">
<li><p>Determine whether</p>
<p><span class="math display">\[
\begin{bmatrix} 1 &amp; 2 \\ 3 &amp; 6 \end{bmatrix}
\]</span></p>
<p>is invertible. Explain both geometrically and algebraically.</p></li>
<li><p>For</p>
<p><span class="math display">\[
\begin{bmatrix} 1 &amp; 0 &amp; 0 \\ 0 &amp; 2 &amp; 0 \\ 0 &amp; 0 &amp; 0 \end{bmatrix},
\]</span></p>
<p>compute the determinant and describe the geometric transformation.</p></li>
<li><p>Challenge: Show that if <span class="math inline">\(\det(A)=0\)</span>, the rows (or columns) of <span class="math inline">\(A\)</span> are linearly dependent.</p></li>
</ol>
<p>The determinant acts as the ultimate yes-or-no test: nonzero means full-dimensional, reversible transformation; zero means collapse and irreversibility.</p>
</section>
</section>
<section id="cofactor-expansion" class="level3">
<h3 class="anchored" data-anchor-id="cofactor-expansion">57. Cofactor Expansion</h3>
<p>While elimination gives a practical way to compute determinants, the cofactor expansion (also called Laplace expansion) offers a recursive definition that works for all square matrices. It expresses the determinant of an <span class="math inline">\(n \times n\)</span> matrix in terms of determinants of smaller <span class="math inline">\((n-1) \times (n-1)\)</span> matrices. This method reveals the internal structure of determinants and serves as a bridge between theory and computation.</p>
<section id="minors-and-cofactors" class="level4">
<h4 class="anchored" data-anchor-id="minors-and-cofactors">Minors and Cofactors</h4>
<ul>
<li><p>The minor <span class="math inline">\(M_{ij}\)</span> of an entry <span class="math inline">\(a_{ij}\)</span> is the determinant of the submatrix obtained by deleting the <span class="math inline">\(i\)</span>-th row and <span class="math inline">\(j\)</span>-th column from <span class="math inline">\(A\)</span>.</p></li>
<li><p>The cofactor <span class="math inline">\(C_{ij}\)</span> adds a sign factor:</p>
<p><span class="math display">\[
C_{ij} = (-1)^{i+j} M_{ij}.
\]</span></p></li>
</ul>
<p>Thus each entry contributes to the determinant through its cofactor, with alternating signs arranged in a checkerboard pattern:</p>
<p><span class="math display">\[
\begin{bmatrix}
+ &amp; - &amp; + &amp; - &amp; \cdots \\
- &amp; + &amp; - &amp; + &amp; \cdots \\
+ &amp; - &amp; + &amp; - &amp; \cdots \\
\vdots &amp; \vdots &amp; \vdots &amp; \vdots &amp; \ddots
\end{bmatrix}.
\]</span></p>
</section>
<section id="the-expansion-formula" class="level4">
<h4 class="anchored" data-anchor-id="the-expansion-formula">The Expansion Formula</h4>
<p>For any row <span class="math inline">\(i\)</span>:</p>
<p><span class="math display">\[
\det(A) = \sum_{j=1}^n a_{ij} C_{ij}.
\]</span></p>
<p>Or for any column <span class="math inline">\(j\)</span>:</p>
<p><span class="math display">\[
\det(A) = \sum_{i=1}^n a_{ij} C_{ij}.
\]</span></p>
<p>That is, the determinant can be computed by expanding along any row or column.</p>
</section>
<section id="example-33-case-1" class="level4">
<h4 class="anchored" data-anchor-id="example-33-case-1">Example: 3×3 Case</h4>
<p>Let</p>
<p><span class="math display">\[
A = \begin{bmatrix}
a &amp; b &amp; c \\
d &amp; e &amp; f \\
g &amp; h &amp; i
\end{bmatrix}.
\]</span></p>
<p>Expanding along the first row:</p>
<p><span class="math display">\[
\det(A) = a \cdot \det \begin{bmatrix} e &amp; f \\ h &amp; i \end{bmatrix}
- b \cdot \det \begin{bmatrix} d &amp; f \\ g &amp; i \end{bmatrix}
+ c \cdot \det \begin{bmatrix} d &amp; e \\ g &amp; h \end{bmatrix}.
\]</span></p>
<p>Simplify each 2×2 determinant:</p>
<p><span class="math display">\[
= a(ei - fh) - b(di - fg) + c(dh - eg).
\]</span></p>
<p>This matches the familiar expansion formula for 3×3 determinants.</p>
</section>
<section id="why-it-works" class="level4">
<h4 class="anchored" data-anchor-id="why-it-works">Why It Works</h4>
<p>Cofactor expansion follows directly from the multilinearity and alternating rules of determinants:</p>
<ul>
<li>Only one element per row and per column contributes to each term.</li>
<li>Signs alternate because swapping rows/columns reverses orientation.</li>
<li>Recursive expansion reduces the problem size until reaching 2×2 determinants, where the formula is simple.</li>
</ul>
</section>
<section id="computational-complexity" class="level4">
<h4 class="anchored" data-anchor-id="computational-complexity">Computational Complexity</h4>
<ul>
<li>For <span class="math inline">\(n=2\)</span>, expansion is immediate.</li>
<li>For <span class="math inline">\(n=3\)</span>, expansion is manageable.</li>
<li>For large <span class="math inline">\(n\)</span>, expansion is very inefficient: computing <span class="math inline">\(\det(A)\)</span> via cofactors requires <span class="math inline">\(O(n!)\)</span> operations.</li>
</ul>
<p>That’s why in practice, elimination or LU decomposition is preferred. Cofactor expansion is best for theory, proofs, and small matrices.</p>
</section>
<section id="geometric-interpretation-12" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-12">Geometric Interpretation</h4>
<p>Each cofactor corresponds to excluding one direction (row/column), measuring the volume of the remaining sub-parallelotope. The alternating sign keeps track of orientation. Thus the determinant is a weighted combination of contributions from all entries along a chosen row or column.</p>
</section>
<section id="applications-19" class="level4">
<h4 class="anchored" data-anchor-id="applications-19">Applications</h4>
<ol type="1">
<li>Theoretical proofs: Cofactor expansion underlies many determinant identities.</li>
<li>Adjugate matrix: Cofactors form the adjugate used in the explicit formula for matrix inverses.</li>
<li>Eigenvalues: Characteristic polynomials use cofactor expansion.</li>
<li>Geometry: Cofactors describe signed volumes of faces of higher-dimensional shapes.</li>
</ol>
</section>
<section id="why-it-matters-53" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-53">Why It Matters</h4>
<ul>
<li>Cofactor expansion connects determinants across dimensions.</li>
<li>It provides a universal definition independent of row operations.</li>
<li>It explains why determinants behave consistently with volume, orientation, and algebraic rules.</li>
</ul>
</section>
<section id="try-it-yourself-56" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-56">Try It Yourself</h4>
<ol type="1">
<li><p>Expand the determinant of</p>
<p><span class="math display">\[
\begin{bmatrix}
2 &amp; 1 &amp; 3 \\
0 &amp; -1 &amp; 4 \\
1 &amp; 2 &amp; 0
\end{bmatrix}
\]</span></p>
<p>along the first row.</p></li>
<li><p>Compute the same determinant by expanding along the second column. Verify the result matches.</p></li>
<li><p>Show that expanding along two different rows gives the same determinant.</p></li>
<li><p>Challenge: Prove by induction that cofactor expansion works for all <span class="math inline">\(n \times n\)</span> matrices.</p></li>
</ol>
<p>Cofactor expansion is not the fastest method, but it reveals the recursive structure of determinants and explains why they hold their rich algebraic and geometric meaning.</p>
</section>
</section>
<section id="permutations-and-the-sign-of-the-determinant" class="level3">
<h3 class="anchored" data-anchor-id="permutations-and-the-sign-of-the-determinant">58. Permutations and the Sign of the Determinant</h3>
<p>Behind every determinant formula lies a hidden structure: permutations. Determinants can be expressed as a weighted sum over all possible ways of selecting one entry from each row and each column of a matrix. The weight for each selection is determined by the sign of the permutation used. This viewpoint reveals why determinants encode orientation and why their formulas alternate between positive and negative terms.</p>
<section id="the-permutation-definition" class="level4">
<h4 class="anchored" data-anchor-id="the-permutation-definition">The Permutation Definition</h4>
<p>Let <span class="math inline">\(S_n\)</span> denote the set of all permutations of <span class="math inline">\(n\)</span> elements. Each permutation <span class="math inline">\(\sigma \in S_n\)</span> rearranges the numbers <span class="math inline">\(\{1, 2, \ldots, n\}\)</span>.</p>
<p>The determinant of an <span class="math inline">\(n \times n\)</span> matrix <span class="math inline">\(A = [a_{ij}]\)</span> is defined as:</p>
<p><span class="math display">\[
\det(A) = \sum_{\sigma \in S_n} \text{sgn}(\sigma) \prod_{i=1}^n a_{i, \sigma(i)}.
\]</span></p>
<ul>
<li>Each product <span class="math inline">\(\prod_{i=1}^n a_{i, \sigma(i)}\)</span> picks one entry from each row and each column, according to <span class="math inline">\(\sigma\)</span>.</li>
<li>The factor <span class="math inline">\(\text{sgn}(\sigma)\)</span> is <span class="math inline">\(+1\)</span> if <span class="math inline">\(\sigma\)</span> is an even permutation (achieved by an even number of swaps), and <span class="math inline">\(-1\)</span> if it is odd.</li>
</ul>
</section>
<section id="why-permutations-appear" class="level4">
<h4 class="anchored" data-anchor-id="why-permutations-appear">Why Permutations Appear</h4>
<p>A determinant requires:</p>
<ol type="1">
<li>Linearity in each row.</li>
<li>Alternating property (row swaps flip the sign).</li>
<li>Normalization (<span class="math inline">\(\det(I)=1\)</span>).</li>
</ol>
<p>When you expand by multilinearity, all possible combinations of choosing one entry per row and column arise. The alternating rule enforces that terms with repeated columns vanish, leaving only permutations. The sign of each permutation enforces the orientation flip.</p>
</section>
<section id="example-22-case-1" class="level4">
<h4 class="anchored" data-anchor-id="example-22-case-1">Example: 2×2 Case</h4>
<p><span class="math display">\[
A = \begin{bmatrix} a &amp; b \\ c &amp; d \end{bmatrix}.
\]</span></p>
<p>There are two permutations in <span class="math inline">\(S_2\)</span>:</p>
<ul>
<li>Identity <span class="math inline">\((1,2)\)</span>: sign <span class="math inline">\(+1\)</span>, contributes <span class="math inline">\(a \cdot d\)</span>.</li>
<li>Swap <span class="math inline">\((2,1)\)</span>: sign <span class="math inline">\(-1\)</span>, contributes <span class="math inline">\(-bc\)</span>.</li>
</ul>
<p>So,</p>
<p><span class="math display">\[
\det(A) = ad - bc.
\]</span></p>
</section>
<section id="example-33-case-2" class="level4">
<h4 class="anchored" data-anchor-id="example-33-case-2">Example: 3×3 Case</h4>
<p><span class="math display">\[
A = \begin{bmatrix} a &amp; b &amp; c \\ d &amp; e &amp; f \\ g &amp; h &amp; i \end{bmatrix}.
\]</span></p>
<p>There are <span class="math inline">\(3! = 6\)</span> permutations:</p>
<ol type="1">
<li><span class="math inline">\((1,2,3)\)</span>: even, <span class="math inline">\(+aei\)</span>.</li>
<li><span class="math inline">\((1,3,2)\)</span>: odd, <span class="math inline">\(-afh\)</span>.</li>
<li><span class="math inline">\((2,1,3)\)</span>: odd, <span class="math inline">\(-bdi\)</span>.</li>
<li><span class="math inline">\((2,3,1)\)</span>: even, <span class="math inline">\(+bfg\)</span>.</li>
<li><span class="math inline">\((3,1,2)\)</span>: even, <span class="math inline">\(+cdh\)</span>.</li>
<li><span class="math inline">\((3,2,1)\)</span>: odd, <span class="math inline">\(-ceg\)</span>.</li>
</ol>
<p>So,</p>
<p><span class="math display">\[
\det(A) = aei + bfg + cdh - ceg - bdi - afh.
\]</span></p>
<p>This is exactly the cofactor expansion result, but now explained as a permutation sum.</p>
</section>
<section id="geometric-meaning-of-signs" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-of-signs">Geometric Meaning of Signs</h4>
<ul>
<li>Even permutations correspond to consistent orientation of basis vectors.</li>
<li>Odd permutations correspond to flipped orientation.</li>
<li>The determinant alternates signs because flipping axes reverses handedness.</li>
</ul>
</section>
<section id="counting-growth" class="level4">
<h4 class="anchored" data-anchor-id="counting-growth">Counting Growth</h4>
<ul>
<li>For <span class="math inline">\(n=4\)</span>, there are <span class="math inline">\(4! = 24\)</span> terms.</li>
<li>For <span class="math inline">\(n=5\)</span>, <span class="math inline">\(5! = 120\)</span> terms.</li>
<li>In general, <span class="math inline">\(n!\)</span> terms make this formula impractical for large matrices.</li>
<li>Still, it gives the deepest definition of determinants, from which all other rules follow.</li>
</ul>
</section>
<section id="applications-20" class="level4">
<h4 class="anchored" data-anchor-id="applications-20">Applications</h4>
<ol type="1">
<li>Abstract algebra: Determinant definition via permutations works over any field.</li>
<li>Combinatorics: Determinants encode signed sums over permutations, connecting to permanents.</li>
<li>Theoretical proofs: Many determinant properties, like multiplicativity, emerge cleanly from the permutation definition.</li>
<li>Leibniz formula: Explicit but impractical formula for computation.</li>
<li>Advanced math: Determinants generalize to alternating multilinear forms in linear algebra and differential geometry.</li>
</ol>
</section>
<section id="why-it-matters-54" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-54">Why It Matters</h4>
<ul>
<li>Provides the most fundamental definition of determinants.</li>
<li>Explains alternating signs in formulas naturally.</li>
<li>Bridges algebra, geometry, and combinatorics.</li>
<li>Shows how orientation emerges from row/column arrangements.</li>
</ul>
</section>
<section id="try-it-yourself-57" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-57">Try It Yourself</h4>
<ol type="1">
<li>Write out all 6 terms in the 3×3 determinant expansion and verify the sign of each permutation.</li>
<li>Compute the determinant of <span class="math inline">\(\begin{bmatrix} 1 &amp; 2 &amp; 3 \\ 4 &amp; 5 &amp; 6 \\ 7 &amp; 8 &amp; 9 \end{bmatrix}\)</span> using the permutation definition.</li>
<li>Show that if two columns are equal, all permutation terms cancel, giving <span class="math inline">\(\det(A)=0\)</span>.</li>
<li>Challenge: Prove that swapping two rows changes the sign of every permutation term, flipping the total determinant.</li>
</ol>
<p>Determinants may look like algebraic puzzles, but the permutation formula reveals their true nature: a grand sum over all possible ways of matching rows to columns, with signs recording whether orientation is preserved or reversed.</p>
</section>
</section>
<section id="cramers-rule" class="level3">
<h3 class="anchored" data-anchor-id="cramers-rule">59. Cramer’s Rule</h3>
<p>Cramer’s Rule is a classical method for solving systems of linear equations using determinants. While rarely used in large-scale computation due to inefficiency, it offers deep theoretical insights into the relationship between determinants, invertibility, and linear systems. It shows how the determinant of a matrix encodes not only volume scaling but also the exact solution to equations.</p>
<section id="the-setup-1" class="level4">
<h4 class="anchored" data-anchor-id="the-setup-1">The Setup</h4>
<p>Consider a system of <span class="math inline">\(n\)</span> linear equations with <span class="math inline">\(n\)</span> unknowns:</p>
<p><span class="math display">\[
Ax = b,
\]</span></p>
<p>where <span class="math inline">\(A\)</span> is an invertible <span class="math inline">\(n \times n\)</span> matrix, <span class="math inline">\(x\)</span> is the vector of unknowns, and <span class="math inline">\(b\)</span> is the right-hand side vector.</p>
<p>Cramer’s Rule states:</p>
<p><span class="math display">\[
x_i = \frac{\det(A_i)}{\det(A)},
\]</span></p>
<p>where <span class="math inline">\(A_i\)</span> is the matrix <span class="math inline">\(A\)</span> with its <span class="math inline">\(i\)</span>-th column replaced by <span class="math inline">\(b\)</span>.</p>
</section>
<section id="example-22-case-2" class="level4">
<h4 class="anchored" data-anchor-id="example-22-case-2">Example: 2×2 Case</h4>
<p>Solve:</p>
<p><span class="math display">\[
\begin{cases}
2x + y = 5 \\
x + 3y = 7
\end{cases}
\]</span></p>
<p>Matrix form:</p>
<p><span class="math display">\[
A = \begin{bmatrix} 2 &amp; 1 \\ 1 &amp; 3 \end{bmatrix}, \quad b = \begin{bmatrix} 5 \\ 7 \end{bmatrix}.
\]</span></p>
<p>Determinant of <span class="math inline">\(A\)</span>:</p>
<p><span class="math display">\[
\det(A) = 2\cdot 3 - 1\cdot 1 = 5.
\]</span></p>
<ul>
<li>For <span class="math inline">\(x_1\)</span>: replace first column with <span class="math inline">\(b\)</span>:</li>
</ul>
<p><span class="math display">\[
A_1 = \begin{bmatrix} 5 &amp; 1 \\ 7 &amp; 3 \end{bmatrix}, \quad \det(A_1) = 15 - 7 = 8.
\]</span></p>
<p>So <span class="math inline">\(x_1 = 8/5\)</span>.</p>
<ul>
<li>For <span class="math inline">\(x_2\)</span>: replace second column with <span class="math inline">\(b\)</span>:</li>
</ul>
<p><span class="math display">\[
A_2 = \begin{bmatrix} 2 &amp; 5 \\ 1 &amp; 7 \end{bmatrix}, \quad \det(A_2) = 14 - 5 = 9.
\]</span></p>
<p>So <span class="math inline">\(x_2 = 9/5\)</span>.</p>
<p>Solution: <span class="math inline">\((x,y) = (8/5, 9/5)\)</span>.</p>
</section>
<section id="why-it-works-1" class="level4">
<h4 class="anchored" data-anchor-id="why-it-works-1">Why It Works</h4>
<p>Since <span class="math inline">\(A\)</span> is invertible,</p>
<p><span class="math display">\[
x = A^{-1}b.
\]</span></p>
<p>But recall the formula for the inverse:</p>
<p><span class="math display">\[
A^{-1} = \frac{1}{\det(A)} \text{adj}(A),
\]</span></p>
<p>where <span class="math inline">\(\text{adj}(A)\)</span> is the adjugate (transpose of the cofactor matrix). When we multiply <span class="math inline">\(\text{adj}(A)b\)</span>, each component naturally becomes a determinant with one column replaced by <span class="math inline">\(b\)</span>. This is exactly Cramer’s Rule.</p>
</section>
<section id="geometric-interpretation-13" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-13">Geometric Interpretation</h4>
<ul>
<li>The denominator <span class="math inline">\(\det(A)\)</span> represents the volume of the parallelotope spanned by the columns of <span class="math inline">\(A\)</span>.</li>
<li>The numerator <span class="math inline">\(\det(A_i)\)</span> represents the volume when the <span class="math inline">\(i\)</span>-th column is replaced by <span class="math inline">\(b\)</span>.</li>
<li>The ratio tells how much of the volume contribution is aligned with the <span class="math inline">\(i\)</span>-th direction, giving the solution coordinate.</li>
</ul>
</section>
<section id="efficiency-and-limitations" class="level4">
<h4 class="anchored" data-anchor-id="efficiency-and-limitations">Efficiency and Limitations</h4>
<ul>
<li>Good for small <span class="math inline">\(n\)</span>: Elegant for 2×2 or 3×3 systems.</li>
<li>Inefficient for large <span class="math inline">\(n\)</span>: Requires computing <span class="math inline">\(n+1\)</span> determinants, each with factorial complexity if done by cofactor expansion.</li>
<li>Numerical instability: Determinants can be sensitive to rounding errors.</li>
<li>In practice, Gaussian elimination or LU decomposition is far superior.</li>
</ul>
</section>
<section id="applications-21" class="level4">
<h4 class="anchored" data-anchor-id="applications-21">Applications</h4>
<ol type="1">
<li>Theoretical proofs: Establishes uniqueness of solutions for small systems.</li>
<li>Geometry: Connects solutions to ratios of volumes of parallelotopes.</li>
<li>Symbolic algebra: Useful for deriving closed-form expressions.</li>
<li>Control theory: Sometimes applied in proofs of controllability/observability.</li>
</ol>
</section>
<section id="why-it-matters-55" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-55">Why It Matters</h4>
<ul>
<li>Provides a clear formula linking determinants and solutions of linear systems.</li>
<li>Demonstrates the power of determinants as more than just volume measures.</li>
<li>Acts as a conceptual bridge between algebraic solutions and geometric interpretations.</li>
</ul>
</section>
<section id="try-it-yourself-58" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-58">Try It Yourself</h4>
<ol type="1">
<li>Solve <span class="math inline">\(\begin{cases} x + 2y = 3 \\ 4x + 5y = 6 \end{cases}\)</span> using Cramer’s Rule.</li>
<li>For the 3×3 system with matrix <span class="math inline">\(\begin{bmatrix} 1 &amp; 2 &amp; 3 \\ 0 &amp; 1 &amp; 4 \\ 5 &amp; 6 &amp; 0 \end{bmatrix}\)</span>, compute <span class="math inline">\(x_1\)</span> using Cramer’s Rule.</li>
<li>Verify that when <span class="math inline">\(\det(A)=0\)</span>, Cramer’s Rule breaks down, matching the fact that the system is either inconsistent or has infinitely many solutions.</li>
<li>Challenge: Derive Cramer’s Rule from the adjugate matrix formula.</li>
</ol>
<p>Cramer’s Rule is not a computational workhorse, but it elegantly ties together determinants, invertibility, and the solution of linear systems-showing how geometry, algebra, and computation meet in one neat formula.</p>
</section>
</section>
<section id="computing-determinants-in-practice" class="level3">
<h3 class="anchored" data-anchor-id="computing-determinants-in-practice">60. Computing Determinants in Practice</h3>
<p>Determinants carry deep meaning, but when it comes to actual computation, the method you choose makes all the difference. For small matrices, formulas like cofactor expansion or Cramer’s Rule are manageable. For larger systems, however, these direct approaches quickly become inefficient. Practical computation relies on systematic algorithms that exploit structure-especially elimination and matrix factorizations.</p>
<section id="small-matrices-n-3" class="level4">
<h4 class="anchored" data-anchor-id="small-matrices-n-3">Small Matrices (n ≤ 3)</h4>
<ul>
<li><p>2×2 case:</p>
<p><span class="math display">\[
\det \begin{bmatrix} a &amp; b \\ c &amp; d \end{bmatrix} = ad - bc.
\]</span></p></li>
<li><p>3×3 case: Either expand by cofactors or use the “rule of Sarrus”:</p>
<p><span class="math display">\[
\det \begin{bmatrix}
a &amp; b &amp; c \\
d &amp; e &amp; f \\
g &amp; h &amp; i
\end{bmatrix} = aei + bfg + cdh - ceg - bdi - afh.
\]</span></p></li>
</ul>
<p>These formulas are compact, but do not generalize well beyond <span class="math inline">\(3 \times 3\)</span>.</p>
</section>
<section id="large-matrices-elimination-and-lu-decomposition" class="level4">
<h4 class="anchored" data-anchor-id="large-matrices-elimination-and-lu-decomposition">Large Matrices: Elimination and LU Decomposition</h4>
<p>For <span class="math inline">\(n &gt; 3\)</span>, practical methods revolve around Gaussian elimination.</p>
<ol type="1">
<li><p>Row Reduction:</p>
<ul>
<li><p>Reduce <span class="math inline">\(A\)</span> to an upper triangular matrix <span class="math inline">\(U\)</span> using row operations.</p></li>
<li><p>Keep track of operations:</p>
<ul>
<li>Row swaps → flip sign of determinant.</li>
<li>Row scaling → multiply determinant by the scaling factor.</li>
<li>Row replacements → no effect.</li>
</ul></li>
<li><p>Once triangular, compute determinant as the product of diagonal entries.</p></li>
</ul></li>
<li><p>LU Factorization:</p>
<ul>
<li>Express <span class="math inline">\(A = LU\)</span>, where <span class="math inline">\(L\)</span> is lower triangular and <span class="math inline">\(U\)</span> is upper triangular.</li>
<li>Then <span class="math inline">\(\det(A) = \det(L)\det(U)\)</span>.</li>
<li>Since <span class="math inline">\(L\)</span> has 1s on its diagonal, <span class="math inline">\(\det(L)=1\)</span>, so the determinant is just the product of diagonals of <span class="math inline">\(U\)</span>.</li>
</ul></li>
</ol>
<p>This approach reduces the complexity to <span class="math inline">\(O(n^3)\)</span>, far more efficient than the factorial growth of cofactor expansion.</p>
</section>
<section id="numerical-considerations" class="level4">
<h4 class="anchored" data-anchor-id="numerical-considerations">Numerical Considerations</h4>
<ul>
<li>Floating-Point Stability: Determinants can be very large or very small, leading to overflow or underflow in computers.</li>
<li>Pivoting: In practice, partial pivoting ensures stability during elimination.</li>
<li>Condition Number: If a matrix is nearly singular (<span class="math inline">\(\det(A)\)</span> close to 0), computed determinants may be highly inaccurate.</li>
</ul>
<p>For these reasons, in numerical linear algebra, determinants are rarely computed directly; instead, properties of LU or QR factorizations are used.</p>
</section>
<section id="determinant-via-eigenvalues" class="level4">
<h4 class="anchored" data-anchor-id="determinant-via-eigenvalues">Determinant via Eigenvalues</h4>
<p>Since the determinant equals the product of eigenvalues,</p>
<p><span class="math display">\[
\det(A) = \lambda_1 \lambda_2 \cdots \lambda_n,
\]</span></p>
<p>it can be computed by finding eigenvalues (numerically via QR iteration or other methods). This is useful when eigenvalues are already needed, but computing them just for the determinant is often more expensive than elimination.</p>
</section>
<section id="special-matrices" class="level4">
<h4 class="anchored" data-anchor-id="special-matrices">Special Matrices</h4>
<ul>
<li>Diagonal or triangular matrices: Determinant is product of diagonals-fastest case.</li>
<li>Block diagonal matrices: Determinant is the product of determinants of blocks.</li>
<li>Sparse matrices: Exploit structure-only nonzero patterns matter.</li>
<li>Orthogonal matrices: Determinant is always <span class="math inline">\(+1\)</span> or <span class="math inline">\(-1\)</span>.</li>
</ul>
</section>
<section id="applications-22" class="level4">
<h4 class="anchored" data-anchor-id="applications-22">Applications</h4>
<ol type="1">
<li>System solving: Determinants test invertibility, but actual solving uses elimination.</li>
<li>Computer graphics: Determinants detect orientation flips (useful for rendering).</li>
<li>Optimization: Determinants of Hessians signal curvature and stability.</li>
<li>Statistics: Determinants of covariance matrices measure uncertainty volumes.</li>
<li>Physics: Determinants appear in Jacobians for change of variables in integrals.</li>
</ol>
</section>
<section id="why-it-matters-56" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-56">Why It Matters</h4>
<ul>
<li>Determinants provide a global property of matrices, but computation must be efficient.</li>
<li>Direct expansion is elegant but impractical.</li>
<li>Elimination-based methods balance theory, speed, and reliability, forming the backbone of modern computational linear algebra.</li>
</ul>
</section>
<section id="try-it-yourself-59" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-59">Try It Yourself</h4>
<ol type="1">
<li>Compute the determinant of <span class="math inline">\(\begin{bmatrix} 2 &amp; 1 &amp; 3 \\ 4 &amp; 1 &amp; 7 \\ -2 &amp; 5 &amp; 1 \end{bmatrix}\)</span> using elimination, confirming the diagonal product method.</li>
<li>For a diagonal matrix with entries <span class="math inline">\((2, 3, -1, 5)\)</span>, verify that the determinant is simply their product.</li>
<li>Use LU decomposition to compute the determinant of a <span class="math inline">\(3 \times 3\)</span> matrix of your choice.</li>
<li>Challenge: Show that determinant computation by LU requires only <span class="math inline">\(O(n^3)\)</span> operations, while cofactor expansion requires <span class="math inline">\(O(n!)\)</span>.</li>
</ol>
<p>Determinants are central, but in practice they are best approached with systematic algorithms, where triangular forms and factorizations reveal the answer quickly and reliably.</p>
</section>
<section id="closing-5" class="level4">
<h4 class="anchored" data-anchor-id="closing-5">Closing</h4>
<pre><code>Flatness or fullness,
determinants quietly weigh
depth in every move.</code></pre>
</section>
</section>
</section>
<section id="chapter-7.-eigenvalues-eigenvectors-and-dynamics" class="level2">
<h2 class="anchored" data-anchor-id="chapter-7.-eigenvalues-eigenvectors-and-dynamics">Chapter 7. Eigenvalues, eigenvectors, and dynamics</h2>
<section id="opening-5" class="level4">
<h4 class="anchored" data-anchor-id="opening-5">Opening</h4>
<pre><code>Stillness in motion,
directions that never fade,
time reveals its core.</code></pre>
</section>
<section id="eigenvalues-and-eigenvectors" class="level3">
<h3 class="anchored" data-anchor-id="eigenvalues-and-eigenvectors">61. Eigenvalues and Eigenvectors</h3>
<p>Among all the concepts in linear algebra, few are as central and powerful as eigenvalues and eigenvectors. They reveal the hidden “axes of action” of a linear transformation-directions in space where the transformation behaves in the simplest possible way. Instead of mixing and rotating everything, an eigenvector is left unchanged in direction, scaled only by its corresponding eigenvalue.</p>
<section id="the-core-idea-1" class="level4">
<h4 class="anchored" data-anchor-id="the-core-idea-1">The Core Idea</h4>
<p>Let <span class="math inline">\(A\)</span> be an <span class="math inline">\(n \times n\)</span> matrix. A nonzero vector <span class="math inline">\(v \in \mathbb{R}^n\)</span> is called an eigenvector of <span class="math inline">\(A\)</span> if</p>
<p><span class="math display">\[
Av = \lambda v,
\]</span></p>
<p>for some scalar <span class="math inline">\(\lambda \in \mathbb{R}\)</span> (or <span class="math inline">\(\mathbb{C}\)</span>). The scalar <span class="math inline">\(\lambda\)</span> is the eigenvalue corresponding to <span class="math inline">\(v\)</span>.</p>
<ul>
<li>Eigenvector: A special direction that is preserved by the transformation.</li>
<li>Eigenvalue: The factor by which the eigenvector is stretched or compressed.</li>
</ul>
<p>If <span class="math inline">\(\lambda &gt; 1\)</span>, the eigenvector is stretched. If <span class="math inline">\(0 &lt; \lambda &lt; 1\)</span>, it is compressed. If <span class="math inline">\(\lambda &lt; 0\)</span>, it is flipped in direction and scaled. If <span class="math inline">\(\lambda = 0\)</span>, the vector is flattened to zero.</p>
</section>
<section id="why-they-matter-1" class="level4">
<h4 class="anchored" data-anchor-id="why-they-matter-1">Why They Matter</h4>
<p>Eigenvalues and eigenvectors describe the intrinsic structure of a transformation:</p>
<ul>
<li>They give preferred directions in which the action of the matrix is simplest.</li>
<li>They summarize long-term behavior of repeated applications (e.g., powers of <span class="math inline">\(A\)</span>).</li>
<li>They connect algebra, geometry, and applications in physics, data science, and engineering.</li>
</ul>
</section>
<section id="example-a-simple-2d-case" class="level4">
<h4 class="anchored" data-anchor-id="example-a-simple-2d-case">Example: A Simple 2D Case</h4>
<p>Let</p>
<p><span class="math display">\[
A = \begin{bmatrix} 2 &amp; 0 \\ 0 &amp; 3 \end{bmatrix}.
\]</span></p>
<ul>
<li><p>Applying <span class="math inline">\(A\)</span> to <span class="math inline">\((1,0)\)</span>:</p>
<p><span class="math display">\[
A \begin{bmatrix} 1 \\ 0 \end{bmatrix} = \begin{bmatrix} 2 \\ 0 \end{bmatrix} = 2 \begin{bmatrix} 1 \\ 0 \end{bmatrix}.
\]</span></p>
<p>So <span class="math inline">\((1,0)\)</span> is an eigenvector with eigenvalue <span class="math inline">\(2\)</span>.</p></li>
<li><p>Applying <span class="math inline">\(A\)</span> to <span class="math inline">\((0,1)\)</span>:</p>
<p><span class="math display">\[
A \begin{bmatrix} 0 \\ 1 \end{bmatrix} = \begin{bmatrix} 0 \\ 3 \end{bmatrix} = 3 \begin{bmatrix} 0 \\ 1 \end{bmatrix}.
\]</span></p>
<p>So <span class="math inline">\((0,1)\)</span> is an eigenvector with eigenvalue <span class="math inline">\(3\)</span>.</p></li>
</ul>
<p>Here the eigenvectors align with the coordinate axes, and the eigenvalues are the diagonal entries.</p>
</section>
<section id="general-case-the-eigenvalue-equation" class="level4">
<h4 class="anchored" data-anchor-id="general-case-the-eigenvalue-equation">General Case: The Eigenvalue Equation</h4>
<p>To find eigenvalues, we solve</p>
<p><span class="math display">\[
Av = \lambda v \quad \Leftrightarrow \quad (A - \lambda I)v = 0.
\]</span></p>
<p>For nontrivial <span class="math inline">\(v\)</span>, the matrix <span class="math inline">\((A - \lambda I)\)</span> must be singular:</p>
<p><span class="math display">\[
\det(A - \lambda I) = 0.
\]</span></p>
<p>This determinant expands to the characteristic polynomial, whose roots are the eigenvalues. Eigenvectors come from solving the corresponding null spaces.</p>
</section>
<section id="geometric-interpretation-14" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-14">Geometric Interpretation</h4>
<ul>
<li>Eigenvectors are invariant directions. When you apply <span class="math inline">\(A\)</span>, the vector may stretch or flip, but it does not rotate off its line.</li>
<li>Eigenvalues are scaling factors. They describe how much stretching, shrinking, or flipping happens along that invariant direction.</li>
</ul>
<p>For example:</p>
<ul>
<li>In 2D, an eigenvector might be a line through the origin where the transformation acts as a stretch.</li>
<li>In 3D, planes of shear often have eigenvectors along axes of invariance.</li>
</ul>
</section>
<section id="dynamics-and-repeated-applications" class="level4">
<h4 class="anchored" data-anchor-id="dynamics-and-repeated-applications">Dynamics and Repeated Applications</h4>
<p>One reason eigenvalues are so important is that they describe repeated transformations:</p>
<p><span class="math display">\[
A^k v = \lambda^k v.
\]</span></p>
<p>If you apply <span class="math inline">\(A\)</span> repeatedly to an eigenvector, the result is predictable: just multiply by <span class="math inline">\(\lambda^k\)</span>. This explains stability in dynamical systems, growth in population models, and convergence in Markov chains.</p>
<ul>
<li>If <span class="math inline">\(|\lambda| &lt; 1\)</span>, repeated applications shrink the vector to zero.</li>
<li>If <span class="math inline">\(|\lambda| &gt; 1\)</span>, the vector grows without bound.</li>
<li>If <span class="math inline">\(\lambda = 1\)</span>, the vector stays the same length (though direction may flip if <span class="math inline">\(\lambda=-1\)</span>).</li>
</ul>
</section>
<section id="applications-23" class="level4">
<h4 class="anchored" data-anchor-id="applications-23">Applications</h4>
<ol type="1">
<li>Physics: Vibrations of molecules, quantum energy levels, and resonance all rely on eigenvalues/eigenvectors.</li>
<li>Data Science: Principal Component Analysis (PCA) finds eigenvectors of covariance matrices to detect key directions of variance.</li>
<li>Markov Chains: Steady-state probabilities correspond to eigenvectors with eigenvalue 1.</li>
<li>Differential Equations: Eigenvalues simplify systems of linear ODEs.</li>
<li>Computer Graphics: Transformations like rotations and scalings can be analyzed with eigen-decompositions.</li>
</ol>
</section>
<section id="why-it-matters-57" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-57">Why It Matters</h4>
<ul>
<li>Eigenvalues and eigenvectors reduce complex transformations to their simplest components.</li>
<li>They unify algebra (roots of characteristic polynomials), geometry (invariant directions), and applications (stability, resonance, variance).</li>
<li>They are the foundation for diagonalization, SVD, and spectral analysis, which dominate modern applied mathematics.</li>
</ul>
</section>
<section id="try-it-yourself-60" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-60">Try It Yourself</h4>
<ol type="1">
<li>Compute the eigenvalues and eigenvectors of <span class="math inline">\(\begin{bmatrix} 4 &amp; 2 \\ 1 &amp; 3 \end{bmatrix}\)</span>.</li>
<li>For <span class="math inline">\(A = \begin{bmatrix} 0 &amp; 1 \\ -1 &amp; 0 \end{bmatrix}\)</span>, find its eigenvalues. (Hint: they are complex.)</li>
<li>Take a random 2×2 matrix and check if its eigenvectors align with coordinate axes.</li>
<li>Challenge: Prove that eigenvectors corresponding to distinct eigenvalues are linearly independent.</li>
</ol>
<p>Eigenvalues and eigenvectors are the “fingerprints” of a matrix: they capture the essential behavior of a transformation, guiding us to understand stability, dynamics, and structure across countless disciplines.</p>
</section>
</section>
<section id="the-characteristic-polynomial" class="level3">
<h3 class="anchored" data-anchor-id="the-characteristic-polynomial">62. The Characteristic Polynomial</h3>
<p>To uncover the eigenvalues of a matrix, we use a central tool: the characteristic polynomial. This polynomial encodes the relationship between a matrix and its eigenvalues. The roots of the polynomial are precisely the eigenvalues, making it the algebraic gateway to spectral analysis.</p>
<section id="definition-3" class="level4">
<h4 class="anchored" data-anchor-id="definition-3">Definition</h4>
<p>For a square matrix <span class="math inline">\(A \in \mathbb{R}^{n \times n}\)</span>, the characteristic polynomial is defined as</p>
<p><span class="math display">\[
p_A(\lambda) = \det(A - \lambda I).
\]</span></p>
<ul>
<li><span class="math inline">\(I\)</span> is the identity matrix of the same size as <span class="math inline">\(A\)</span>.</li>
<li>The polynomial <span class="math inline">\(p_A(\lambda)\)</span> has degree <span class="math inline">\(n\)</span>.</li>
<li>The eigenvalues of <span class="math inline">\(A\)</span> are exactly the roots of <span class="math inline">\(p_A(\lambda)\)</span>.</li>
</ul>
</section>
<section id="why-this-works-1" class="level4">
<h4 class="anchored" data-anchor-id="why-this-works-1">Why This Works</h4>
<p>The eigenvalue equation is</p>
<p><span class="math display">\[
Av = \lambda v \quad \iff \quad (A - \lambda I)v = 0.
\]</span></p>
<p>For nontrivial <span class="math inline">\(v\)</span>, the matrix <span class="math inline">\(A - \lambda I\)</span> must be singular:</p>
<p><span class="math display">\[
\det(A - \lambda I) = 0.
\]</span></p>
<p>Thus, eigenvalues are precisely the scalars <span class="math inline">\(\lambda\)</span> for which the determinant vanishes.</p>
</section>
<section id="example-22-case-3" class="level4">
<h4 class="anchored" data-anchor-id="example-22-case-3">Example: 2×2 Case</h4>
<p>Let</p>
<p><span class="math display">\[
A = \begin{bmatrix} 4 &amp; 2 \\ 1 &amp; 3 \end{bmatrix}.
\]</span></p>
<p>Compute:</p>
<p><span class="math display">\[
p_A(\lambda) = \det \begin{bmatrix} 4-\lambda &amp; 2 \\ 1 &amp; 3-\lambda \end{bmatrix}.
\]</span></p>
<p>Expanding:</p>
<p><span class="math display">\[
p_A(\lambda) = (4-\lambda)(3-\lambda) - 2.
\]</span></p>
<p><span class="math display">\[
= \lambda^2 - 7\lambda + 10.
\]</span></p>
<p>The roots are <span class="math inline">\(\lambda = 5\)</span> and <span class="math inline">\(\lambda = 2\)</span>. These are the eigenvalues of <span class="math inline">\(A\)</span>.</p>
</section>
<section id="example-33-case-3" class="level4">
<h4 class="anchored" data-anchor-id="example-33-case-3">Example: 3×3 Case</h4>
<p>For</p>
<p><span class="math display">\[
B = \begin{bmatrix} 2 &amp; 0 &amp; 0 \\ 0 &amp; 3 &amp; 4 \\ 0 &amp; 4 &amp; 9 \end{bmatrix},
\]</span></p>
<p><span class="math display">\[
p_B(\lambda) = \det \begin{bmatrix} 2-\lambda &amp; 0 &amp; 0 \\ 0 &amp; 3-\lambda &amp; 4 \\ 0 &amp; 4 &amp; 9-\lambda \end{bmatrix}.
\]</span></p>
<p>Expand:</p>
<p><span class="math display">\[
p_B(\lambda) = (2-\lambda)\big[(3-\lambda)(9-\lambda) - 16\big].
\]</span></p>
<p><span class="math display">\[
= (2-\lambda)(\lambda^2 - 12\lambda + 11).
\]</span></p>
<p>Roots: <span class="math inline">\(\lambda = 2, 1, 11\)</span>.</p>
</section>
<section id="properties-of-the-characteristic-polynomial" class="level4">
<h4 class="anchored" data-anchor-id="properties-of-the-characteristic-polynomial">Properties of the Characteristic Polynomial</h4>
<ol type="1">
<li>Degree: Always degree <span class="math inline">\(n\)</span>.</li>
<li>Leading term: <span class="math inline">\((-1)^n \lambda^n\)</span>.</li>
<li>Constant term: <span class="math inline">\(\det(A)\)</span>.</li>
<li>Coefficient of <span class="math inline">\(\lambda^{n-1}\)</span>: <span class="math inline">\(-\text{tr}(A)\)</span>, where <span class="math inline">\(\text{tr}(A)\)</span> is the trace (sum of diagonal entries).</li>
</ol>
<p>So:</p>
<p><span class="math display">\[
p_A(\lambda) = (-1)^n \lambda^n + (\text{tr}(A))(-1)^{n-1}\lambda^{n-1} + \cdots + \det(A).
\]</span></p>
<p>This ties together trace, determinant, and eigenvalues in one polynomial.</p>
</section>
<section id="geometric-meaning-10" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-10">Geometric Meaning</h4>
<ul>
<li>The roots of the characteristic polynomial tell us scaling factors along invariant directions.</li>
<li>In 2D: the polynomial encodes area scaling (<span class="math inline">\(\det(A)\)</span>) and total stretching (<span class="math inline">\(\text{tr}(A)\)</span>).</li>
<li>In higher dimensions: it condenses the complexity of <span class="math inline">\(A\)</span> into a single equation whose solutions reveal the spectrum.</li>
</ul>
</section>
<section id="applications-24" class="level4">
<h4 class="anchored" data-anchor-id="applications-24">Applications</h4>
<ol type="1">
<li>Eigenvalue computation: Foundation for diagonalization and spectral theory.</li>
<li>Control theory: Stability of systems depends on eigenvalues (roots of the characteristic polynomial).</li>
<li>Differential equations: Characteristic polynomials describe natural frequencies and modes of oscillation.</li>
<li>Graph theory: The characteristic polynomial of an adjacency matrix encodes structural properties of the graph.</li>
<li>Quantum mechanics: Energy levels of quantum systems come from solving characteristic polynomials of operators.</li>
</ol>
</section>
<section id="why-it-matters-58" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-58">Why It Matters</h4>
<ul>
<li>Provides a systematic, algebraic way to find eigenvalues.</li>
<li>Connects trace and determinant to deeper spectral properties.</li>
<li>Bridges linear algebra, polynomial theory, and geometry.</li>
<li>Forms the foundation for modern computational methods like QR iteration.</li>
</ul>
</section>
<section id="try-it-yourself-61" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-61">Try It Yourself</h4>
<ol type="1">
<li>Compute the characteristic polynomial of <span class="math inline">\(\begin{bmatrix} 1 &amp; 1 \\ 0 &amp; 2 \end{bmatrix}\)</span>. Find its eigenvalues.</li>
<li>Verify that the product of eigenvalues equals the determinant.</li>
<li>Verify that the sum of eigenvalues equals the trace.</li>
<li>Challenge: Prove that <span class="math inline">\(p_{AB}(\lambda) = p_{BA}(\lambda)\)</span> for any <span class="math inline">\(A, B\)</span> of the same size.</li>
</ol>
<p>The characteristic polynomial distills a matrix into a single algebraic object whose roots reveal the essential dynamics of the transformation.</p>
</section>
</section>
<section id="algebraic-vs.-geometric-multiplicity" class="level3">
<h3 class="anchored" data-anchor-id="algebraic-vs.-geometric-multiplicity">63. Algebraic vs.&nbsp;Geometric Multiplicity</h3>
<p>When studying eigenvalues, it’s not enough to just find the roots of the characteristic polynomial. Each eigenvalue can appear multiple times, and this “multiplicity” can be understood in two distinct but related ways: algebraic multiplicity (how many times it appears as a root) and geometric multiplicity (the dimension of its eigenspace). These two multiplicities capture both the algebraic and geometric richness of eigenvalues.</p>
<section id="algebraic-multiplicity" class="level4">
<h4 class="anchored" data-anchor-id="algebraic-multiplicity">Algebraic Multiplicity</h4>
<p>The algebraic multiplicity (AM) of an eigenvalue <span class="math inline">\(\lambda\)</span> is the number of times it appears as a root of the characteristic polynomial <span class="math inline">\(p_A(\lambda)\)</span>.</p>
<ul>
<li>If <span class="math inline">\((\lambda - \lambda_0)^k\)</span> divides <span class="math inline">\(p_A(\lambda)\)</span>, then the algebraic multiplicity of <span class="math inline">\(\lambda_0\)</span> is <span class="math inline">\(k\)</span>.</li>
<li>The sum of all algebraic multiplicities equals the size of the matrix (<span class="math inline">\(n\)</span>).</li>
</ul>
<p>Example: If</p>
<p><span class="math display">\[
p_A(\lambda) = (\lambda-2)^3(\lambda+1)^2,
\]</span></p>
<p>then eigenvalue <span class="math inline">\(\lambda=2\)</span> has AM = 3, and <span class="math inline">\(\lambda=-1\)</span> has AM = 2.</p>
</section>
<section id="geometric-multiplicity" class="level4">
<h4 class="anchored" data-anchor-id="geometric-multiplicity">Geometric Multiplicity</h4>
<p>The geometric multiplicity (GM) of an eigenvalue <span class="math inline">\(\lambda\)</span> is the dimension of the eigenspace corresponding to <span class="math inline">\(\lambda\)</span>:</p>
<p><span class="math display">\[
\text{GM}(\lambda) = \dim(\ker(A - \lambda I)).
\]</span></p>
<ul>
<li><p>This counts how many linearly independent eigenvectors correspond to <span class="math inline">\(\lambda\)</span>.</p></li>
<li><p>Always satisfies:</p>
<p><span class="math display">\[
1 \leq \text{GM}(\lambda) \leq \text{AM}(\lambda).
\]</span></p></li>
</ul>
<p>Example: If</p>
<p><span class="math display">\[
A = \begin{bmatrix} 2 &amp; 1 \\ 0 &amp; 2 \end{bmatrix},
\]</span></p>
<p>then <span class="math inline">\(p_A(\lambda) = (\lambda-2)^2\)</span>.</p>
<ul>
<li><p>AM of <span class="math inline">\(\lambda=2\)</span> is 2.</p></li>
<li><p>Solve <span class="math inline">\((A-2I)v=0\)</span>:</p>
<p><span class="math display">\[
\begin{bmatrix} 0 &amp; 1 \\ 0 &amp; 0 \end{bmatrix} v = 0 \quad \Rightarrow \quad v = \begin{bmatrix} 1 \\ 0 \end{bmatrix}.
\]</span></p>
<p>Only 1 independent eigenvector.</p></li>
<li><p>GM of <span class="math inline">\(\lambda=2\)</span> is 1.</p></li>
</ul>
</section>
<section id="relationship-between-the-two" class="level4">
<h4 class="anchored" data-anchor-id="relationship-between-the-two">Relationship Between the Two</h4>
<ul>
<li>Always: <span class="math inline">\(\text{GM}(\lambda) \leq \text{AM}(\lambda)\)</span>.</li>
<li>If they are equal for all eigenvalues, the matrix is diagonalizable.</li>
<li>If GM &lt; AM for some eigenvalue, the matrix is defective, meaning it cannot be diagonalized, though it may still have a Jordan canonical form.</li>
</ul>
</section>
<section id="geometric-meaning-11" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-11">Geometric Meaning</h4>
<ul>
<li>AM measures how strongly the eigenvalue is “encoded” in the polynomial.</li>
<li>GM measures how much geometric freedom the eigenvalue’s eigenspace provides.</li>
<li>If AM &gt; GM, the eigenvalue “wants” more independent directions than the space allows.</li>
</ul>
<p>Think of AM as the <em>theoretical demand</em> for eigenvectors, and GM as the <em>actual supply</em>.</p>
</section>
<section id="example-diagonalizable-vs.-defective" class="level4">
<h4 class="anchored" data-anchor-id="example-diagonalizable-vs.-defective">Example: Diagonalizable vs.&nbsp;Defective</h4>
<ol type="1">
<li><p>Diagonalizable case:</p>
<p><span class="math display">\[
B = \begin{bmatrix} 2 &amp; 0 \\ 0 &amp; 2 \end{bmatrix}.
\]</span></p>
<ul>
<li><span class="math inline">\(p_B(\lambda) = (\lambda-2)^2\)</span>.</li>
<li>AM = 2 for eigenvalue 2.</li>
<li>GM = 2, since the eigenspace is all of <span class="math inline">\(\mathbb{R}^2\)</span>.</li>
<li>Enough eigenvectors to diagonalize.</li>
</ul></li>
<li><p>Defective case: The earlier example</p>
<p><span class="math display">\[
A = \begin{bmatrix} 2 &amp; 1 \\ 0 &amp; 2 \end{bmatrix}
\]</span></p>
<p>had AM = 2, GM = 1.</p>
<ul>
<li>Not enough eigenvectors.</li>
<li>Cannot be diagonalized.</li>
</ul></li>
</ol>
</section>
<section id="applications-25" class="level4">
<h4 class="anchored" data-anchor-id="applications-25">Applications</h4>
<ol type="1">
<li>Diagonalization: Only possible when GM = AM for all eigenvalues.</li>
<li>Jordan form: Defective matrices require Jordan blocks, governed by the gap between AM and GM.</li>
<li>Differential equations: The solution form depends on multiplicity; repeated eigenvalues with fewer eigenvectors require generalized solutions.</li>
<li>Stability analysis: Multiplicities reveal degeneracies in dynamical systems.</li>
<li>Quantum mechanics: Degeneracy of eigenvalues (AM vs.&nbsp;GM) encodes physical symmetry.</li>
</ol>
</section>
<section id="why-it-matters-59" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-59">Why It Matters</h4>
<ul>
<li>Multiplicities separate algebraic roots from geometric structure.</li>
<li>They decide whether diagonalization is possible.</li>
<li>They reveal hidden constraints in systems with repeated eigenvalues.</li>
<li>They form the basis for advanced concepts like Jordan canonical form and generalized eigenvectors.</li>
</ul>
</section>
<section id="try-it-yourself-62" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-62">Try It Yourself</h4>
<ol type="1">
<li>Find AM and GM for <span class="math inline">\(\begin{bmatrix} 3 &amp; 1 \\ 0 &amp; 3 \end{bmatrix}\)</span>.</li>
<li>Find AM and GM for <span class="math inline">\(\begin{bmatrix} 3 &amp; 0 \\ 0 &amp; 3 \end{bmatrix}\)</span>. Compare with the first case.</li>
<li>Show that AM always equals the multiplicity of a root of the characteristic polynomial.</li>
<li>Challenge: Prove that for any eigenvalue, GM ≥ 1.</li>
</ol>
<p>Algebraic and geometric multiplicity together tell the full story: the algebra tells us how many times an eigenvalue appears, while the geometry tells us how much room it really occupies in the vector space.</p>
</section>
</section>
<section id="diagonalization" class="level3">
<h3 class="anchored" data-anchor-id="diagonalization">64. Diagonalization</h3>
<p>Diagonalization is one of the most powerful ideas in linear algebra. It takes a complicated matrix and, when possible, rewrites it in a simple form where its action is completely transparent. A diagonal matrix is easy to understand: it just stretches or compresses each coordinate axis by a fixed factor. If we can transform a matrix into a diagonal one, many calculations-like computing powers or exponentials-become almost trivial.</p>
<section id="the-core-concept" class="level4">
<h4 class="anchored" data-anchor-id="the-core-concept">The Core Concept</h4>
<p>A square matrix <span class="math inline">\(A \in \mathbb{R}^{n \times n}\)</span> is diagonalizable if there exists an invertible matrix <span class="math inline">\(P\)</span> and a diagonal matrix <span class="math inline">\(D\)</span> such that</p>
<p><span class="math display">\[
A = P D P^{-1}.
\]</span></p>
<ul>
<li>The diagonal entries of <span class="math inline">\(D\)</span> are the eigenvalues of <span class="math inline">\(A\)</span>.</li>
<li>The columns of <span class="math inline">\(P\)</span> are the corresponding eigenvectors.</li>
</ul>
<p>In words: <span class="math inline">\(A\)</span> can be “rewritten” in a coordinate system made of its eigenvectors, where its action reduces to simple scaling along independent directions.</p>
</section>
<section id="why-diagonalization-matters" class="level4">
<h4 class="anchored" data-anchor-id="why-diagonalization-matters">Why Diagonalization Matters</h4>
<ol type="1">
<li><p>Simplifies Computations:</p>
<ul>
<li><p>Computing powers:</p>
<p><span class="math display">\[
A^k = P D^k P^{-1}, \quad D^k \text{ is trivial to compute}.
\]</span></p></li>
<li><p>Matrix exponential:</p>
<p><span class="math display">\[
e^A = P e^D P^{-1}.
\]</span></p>
<p>Critical in solving differential equations.</p></li>
</ul></li>
<li><p>Clarifies Dynamics:</p>
<ul>
<li>Long-term behavior of iterative processes depends directly on eigenvalues.</li>
<li>Stable vs.&nbsp;unstable systems can be read off from <span class="math inline">\(D\)</span>.</li>
</ul></li>
<li><p>Reveals Structure:</p>
<ul>
<li>Tells us whether the system can be understood through independent modes.</li>
<li>Connects algebraic structure with geometry.</li>
</ul></li>
</ol>
</section>
<section id="conditions-for-diagonalization" class="level4">
<h4 class="anchored" data-anchor-id="conditions-for-diagonalization">Conditions for Diagonalization</h4>
<p>A matrix <span class="math inline">\(A\)</span> is diagonalizable if and only if it has enough linearly independent eigenvectors to form a basis for <span class="math inline">\(\mathbb{R}^n\)</span>.</p>
<ul>
<li>Equivalently: For each eigenvalue, geometric multiplicity = algebraic multiplicity.</li>
<li>Distinct eigenvalues guarantee diagonalizability, since their eigenvectors are linearly independent.</li>
</ul>
</section>
<section id="example-diagonalizable-case" class="level4">
<h4 class="anchored" data-anchor-id="example-diagonalizable-case">Example: Diagonalizable Case</h4>
<p>Let</p>
<p><span class="math display">\[
A = \begin{bmatrix} 4 &amp; 0 \\ 1 &amp; 3 \end{bmatrix}.
\]</span></p>
<ul>
<li><p>Characteristic polynomial:</p>
<p><span class="math display">\[
p_A(\lambda) = (4-\lambda)(3-\lambda).
\]</span></p>
<p>Eigenvalues: <span class="math inline">\(\lambda_1=4, \lambda_2=3\)</span>.</p></li>
<li><p>Eigenvectors:</p>
<ul>
<li>For <span class="math inline">\(\lambda=4\)</span>: <span class="math inline">\((1,1)^T\)</span>.</li>
<li>For <span class="math inline">\(\lambda=3\)</span>: <span class="math inline">\((0,1)^T\)</span>.</li>
</ul></li>
<li><p>Build <span class="math inline">\(P = \begin{bmatrix} 1 &amp; 0 \\ 1 &amp; 1 \end{bmatrix}\)</span>, <span class="math inline">\(D = \begin{bmatrix} 4 &amp; 0 \\ 0 &amp; 3 \end{bmatrix}\)</span>.</p></li>
<li><p>Then <span class="math inline">\(A = P D P^{-1}\)</span>.</p></li>
</ul>
<p>Now, computing <span class="math inline">\(A^{10}\)</span> is easy: just compute <span class="math inline">\(D^{10}\)</span> and conjugate.</p>
</section>
<section id="example-defective-non-diagonalizable-case" class="level4">
<h4 class="anchored" data-anchor-id="example-defective-non-diagonalizable-case">Example: Defective (Non-Diagonalizable) Case</h4>
<p><span class="math display">\[
B = \begin{bmatrix} 2 &amp; 1 \\ 0 &amp; 2 \end{bmatrix}.
\]</span></p>
<ul>
<li>Characteristic polynomial: <span class="math inline">\((\lambda - 2)^2\)</span>.</li>
<li>AM of eigenvalue 2 is 2, but GM = 1 (only one eigenvector).</li>
<li>Not diagonalizable. Needs Jordan form instead.</li>
</ul>
</section>
<section id="geometric-meaning-12" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-12">Geometric Meaning</h4>
<p>Diagonalization means we can rotate into a basis of eigenvectors where the transformation acts simply: scale each axis by its eigenvalue.</p>
<ul>
<li>Think of a room where the floor stretches more in one direction than another. In the right coordinate system (aligned with eigenvectors), the stretch is purely along axes.</li>
<li>Without diagonalization, stretching mixes directions and is harder to describe.</li>
</ul>
</section>
<section id="applications-26" class="level4">
<h4 class="anchored" data-anchor-id="applications-26">Applications</h4>
<ol type="1">
<li>Differential Equations: Solving systems of linear ODEs relies on diagonalization or Jordan form.</li>
<li>Markov Chains: Transition matrices are analyzed through diagonalization to study steady states.</li>
<li>Quantum Mechanics: Operators are diagonalized to reveal measurable states.</li>
<li>PCA (Principal Component Analysis): A covariance matrix is diagonalized to extract independent variance directions.</li>
<li>Computer Graphics: Diagonalization simplifies rotation-scaling transformations.</li>
</ol>
</section>
<section id="why-it-matters-60" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-60">Why It Matters</h4>
<p>Diagonalization transforms complexity into simplicity. It exposes the fundamental action of a matrix: scaling along preferred axes. Without it, understanding or computing repeated transformations would be intractable.</p>
</section>
<section id="try-it-yourself-63" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-63">Try It Yourself</h4>
<ol type="1">
<li><p>Diagonalize</p>
<p><span class="math display">\[
C = \begin{bmatrix} 1 &amp; 1 \\ 0 &amp; 2 \end{bmatrix}.
\]</span></p>
<p>Compute <span class="math inline">\(C^5\)</span> using <span class="math inline">\(P D^5 P^{-1}\)</span>.</p></li>
<li><p>Show why</p>
<p><span class="math display">\[
\begin{bmatrix} 2 &amp; 1 \\ 0 &amp; 2 \end{bmatrix}
\]</span></p>
<p>cannot be diagonalized.</p></li>
<li><p>Challenge: Prove that any symmetric real matrix is diagonalizable with an orthogonal basis.</p></li>
</ol>
<p>Diagonalization is like finding the natural “language” of a matrix: once we listen in its native basis, everything becomes clear, elegant, and simple.</p>
</section>
</section>
<section id="powers-of-a-matrix" class="level3">
<h3 class="anchored" data-anchor-id="powers-of-a-matrix">65. Powers of a Matrix</h3>
<p>Once we know about diagonalization, one of its most powerful consequences is the ability to compute powers of a matrix efficiently. Normally, multiplying a matrix by itself repeatedly is expensive and messy. But if a matrix can be diagonalized, its powers become almost trivial to calculate. This is crucial in understanding long-term behavior of dynamical systems, Markov chains, and iterative algorithms.</p>
<section id="the-general-principle" class="level4">
<h4 class="anchored" data-anchor-id="the-general-principle">The General Principle</h4>
<p>If a matrix <span class="math inline">\(A\)</span> is diagonalizable, then</p>
<p><span class="math display">\[
A = P D P^{-1},
\]</span></p>
<p>where <span class="math inline">\(D\)</span> is diagonal and <span class="math inline">\(P\)</span> is invertible.</p>
<p>Then for any positive integer <span class="math inline">\(k\)</span>:</p>
<p><span class="math display">\[
A^k = (P D P^{-1})^k = P D^k P^{-1}.
\]</span></p>
<p>Because <span class="math inline">\(P^{-1}P = I\)</span>, the middle terms cancel out in the product.</p>
<ul>
<li>Computing <span class="math inline">\(D^k\)</span> is simple: just raise each diagonal entry to the <span class="math inline">\(k\)</span>-th power.</li>
<li>Thus, eigenvalues control the growth or decay of powers of the matrix.</li>
</ul>
</section>
<section id="example-a-simple-diagonal-case" class="level4">
<h4 class="anchored" data-anchor-id="example-a-simple-diagonal-case">Example: A Simple Diagonal Case</h4>
<p>Let</p>
<p><span class="math display">\[
D = \begin{bmatrix} 2 &amp; 0 \\ 0 &amp; 3 \end{bmatrix}.
\]</span></p>
<p>Then</p>
<p><span class="math display">\[
D^k = \begin{bmatrix} 2^k &amp; 0 \\ 0 &amp; 3^k \end{bmatrix}.
\]</span></p>
<p>Each eigenvalue is raised independently to the <span class="math inline">\(k\)</span>-th power.</p>
</section>
<section id="example-using-diagonalization" class="level4">
<h4 class="anchored" data-anchor-id="example-using-diagonalization">Example: Using Diagonalization</h4>
<p>Consider</p>
<p><span class="math display">\[
A = \begin{bmatrix} 4 &amp; 0 \\ 1 &amp; 3 \end{bmatrix}.
\]</span></p>
<p>From before, we know it diagonalizes as</p>
<p><span class="math display">\[
A = P D P^{-1}, \quad D = \begin{bmatrix} 4 &amp; 0 \\ 0 &amp; 3 \end{bmatrix}.
\]</span></p>
<p>So,</p>
<p><span class="math display">\[
A^k = P \begin{bmatrix} 4^k &amp; 0 \\ 0 &amp; 3^k \end{bmatrix} P^{-1}.
\]</span></p>
<p>Instead of multiplying <span class="math inline">\(A\)</span> by itself <span class="math inline">\(k\)</span> times, we just exponentiate the eigenvalues.</p>
</section>
<section id="long-term-behavior" class="level4">
<h4 class="anchored" data-anchor-id="long-term-behavior">Long-Term Behavior</h4>
<p>Eigenvalues reveal exactly what happens as <span class="math inline">\(k \to \infty\)</span>.</p>
<ul>
<li>If all eigenvalues satisfy <span class="math inline">\(|\lambda| &lt; 1\)</span>, then <span class="math inline">\(A^k \to 0\)</span>.</li>
<li>If some eigenvalues have <span class="math inline">\(|\lambda| &gt; 1\)</span>, then <span class="math inline">\(A^k\)</span> diverges along those eigenvector directions.</li>
<li>If <span class="math inline">\(|\lambda| = 1\)</span>, the behavior depends on the specific structure: it may oscillate, stabilize, or remain bounded.</li>
</ul>
<p>This explains stability in recursive systems and iterative algorithms.</p>
</section>
<section id="special-case-markov-chains" class="level4">
<h4 class="anchored" data-anchor-id="special-case-markov-chains">Special Case: Markov Chains</h4>
<p>In probability, the transition matrix of a Markov chain has eigenvalues less than or equal to 1.</p>
<ul>
<li>The largest eigenvalue is always <span class="math inline">\(\lambda = 1\)</span>.</li>
<li>As powers of the transition matrix grow, the chain converges to the eigenvector associated with <span class="math inline">\(\lambda = 1\)</span>, representing the stationary distribution.</li>
</ul>
<p>Thus, <span class="math inline">\(A^k\)</span> describes the long-run behavior of the chain.</p>
</section>
<section id="non-diagonalizable-matrices" class="level4">
<h4 class="anchored" data-anchor-id="non-diagonalizable-matrices">Non-Diagonalizable Matrices</h4>
<p>If a matrix is not diagonalizable, things become more complicated. Such matrices require the Jordan canonical form, where blocks can lead to terms like <span class="math inline">\(k \lambda^{k-1}\)</span>.</p>
<p>Example:</p>
<p><span class="math display">\[
B = \begin{bmatrix} 2 &amp; 1 \\ 0 &amp; 2 \end{bmatrix}.
\]</span></p>
<p>Then</p>
<p><span class="math display">\[
B^k = \begin{bmatrix} 2^k &amp; k 2^{k-1} \\ 0 &amp; 2^k \end{bmatrix}.
\]</span></p>
<p>The presence of the off-diagonal entry introduces linear growth in <span class="math inline">\(k\)</span>, in addition to exponential scaling.</p>
</section>
<section id="geometric-meaning-13" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-13">Geometric Meaning</h4>
<ul>
<li>Powers of <span class="math inline">\(A\)</span> correspond to repeated application of the linear transformation.</li>
<li>Eigenvalues dictate whether directions expand, shrink, or remain steady.</li>
<li>The eigenvectors mark the axes along which the repeated action is simplest to describe.</li>
</ul>
<p>Think of stretching a rubber sheet: after each stretch, the sheet aligns more and more strongly with the dominant eigenvector.</p>
</section>
<section id="applications-27" class="level4">
<h4 class="anchored" data-anchor-id="applications-27">Applications</h4>
<ol type="1">
<li>Dynamical Systems: Population models, economic growth, and iterative algorithms all rely on powers of a matrix.</li>
<li>Markov Chains: Powers reveal equilibrium behavior and mixing rates.</li>
<li>Differential Equations: Discrete-time models use matrix powers to describe state evolution.</li>
<li>Computer Graphics: Repeated transformations can be analyzed via eigenvalues.</li>
<li>Machine Learning: Convergence of iterative solvers (like gradient descent with linear updates) depends on spectral radius.</li>
</ol>
</section>
<section id="why-it-matters-61" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-61">Why It Matters</h4>
<p>Matrix powers are the foundation of stability analysis, asymptotic behavior, and convergence. Diagonalization turns this from a brute-force multiplication into a deep, structured understanding.</p>
</section>
<section id="try-it-yourself-64" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-64">Try It Yourself</h4>
<ol type="1">
<li>Compute <span class="math inline">\(A^5\)</span> for <span class="math inline">\(\begin{bmatrix} 2 &amp; 0 \\ 0 &amp; 3 \end{bmatrix}\)</span>.</li>
<li>For <span class="math inline">\(\begin{bmatrix} 1 &amp; 1 \\ 0 &amp; 1 \end{bmatrix}\)</span>, compute <span class="math inline">\(A^k\)</span>. What happens as <span class="math inline">\(k \to \infty\)</span>?</li>
<li>Explore what happens to <span class="math inline">\(A^k\)</span> when the largest eigenvalue has absolute value &lt; 1, = 1, and &gt; 1.</li>
<li>Challenge: Show that if a diagonalizable matrix has eigenvalues <span class="math inline">\(|\lambda_i| &lt; 1\)</span>, then <span class="math inline">\(\lim_{k \to \infty} A^k = 0\)</span>.</li>
</ol>
<p>Powers of a matrix reveal the story of repetition: how a transformation evolves when applied again and again. They connect linear algebra to time, growth, and stability in every system that unfolds step by step.</p>
</section>
</section>
<section id="real-vs.-complex-spectra" class="level3">
<h3 class="anchored" data-anchor-id="real-vs.-complex-spectra">66. Real vs.&nbsp;Complex Spectra</h3>
<p>Not all eigenvalues are real numbers. Even when working with real matrices, eigenvalues can emerge as complex numbers. Understanding when eigenvalues are real, when they are complex, and what this means geometrically is critical for grasping the full behavior of linear transformations.</p>
<section id="eigenvalues-over-the-complex-numbers" class="level4">
<h4 class="anchored" data-anchor-id="eigenvalues-over-the-complex-numbers">Eigenvalues Over the Complex Numbers</h4>
<p>Every square matrix <span class="math inline">\(A \in \mathbb{R}^{n \times n}\)</span> has at least one eigenvalue in the complex numbers. This is guaranteed by the Fundamental Theorem of Algebra, which says every polynomial (like the characteristic polynomial) has roots in <span class="math inline">\(\mathbb{C}\)</span>.</p>
<ul>
<li>If <span class="math inline">\(p_A(\lambda)\)</span> has only real roots, all eigenvalues are real.</li>
<li>If <span class="math inline">\(p_A(\lambda)\)</span> has quadratic factors with no real roots, then eigenvalues appear as complex conjugate pairs.</li>
</ul>
</section>
<section id="why-complex-numbers-appear" class="level4">
<h4 class="anchored" data-anchor-id="why-complex-numbers-appear">Why Complex Numbers Appear</h4>
<p>Consider a 2D rotation matrix:</p>
<p><span class="math display">\[
R_\theta = \begin{bmatrix} \cos\theta &amp; -\sin\theta \\ \sin\theta &amp; \cos\theta \end{bmatrix}.
\]</span></p>
<p>The characteristic polynomial is</p>
<p><span class="math display">\[
p(\lambda) = \lambda^2 - 2\cos\theta \lambda + 1.
\]</span></p>
<p>The eigenvalues are</p>
<p><span class="math display">\[
\lambda = \cos\theta \pm i \sin\theta = e^{\pm i\theta}.
\]</span></p>
<ul>
<li>Unless <span class="math inline">\(\theta = 0, \pi\)</span>, these eigenvalues are not real.</li>
<li>Geometrically, this makes sense: pure rotation has no invariant real direction. Instead, the eigenvalues are complex numbers of unit magnitude, encoding the rotation angle.</li>
</ul>
</section>
<section id="real-vs.-complex-scenarios" class="level4">
<h4 class="anchored" data-anchor-id="real-vs.-complex-scenarios">Real vs.&nbsp;Complex Scenarios</h4>
<ol type="1">
<li><p>Symmetric Real Matrices:</p>
<ul>
<li>All eigenvalues are real.</li>
<li>Eigenvectors form an orthogonal basis.</li>
<li>Example: <span class="math inline">\(\begin{bmatrix} 2 &amp; 1 \\ 1 &amp; 2 \end{bmatrix}\)</span> has eigenvalues <span class="math inline">\(3, 1\)</span>.</li>
</ul></li>
<li><p>General Real Matrices:</p>
<ul>
<li>Eigenvalues may be complex.</li>
<li>If complex, they always come in conjugate pairs: if <span class="math inline">\(\lambda = a+bi\)</span>, then <span class="math inline">\(\overline{\lambda} = a-bi\)</span> is also an eigenvalue.</li>
</ul></li>
<li><p>Skew-Symmetric Matrices:</p>
<ul>
<li>Purely imaginary eigenvalues.</li>
<li>Example: <span class="math inline">\(\begin{bmatrix} 0 &amp; -1 \\ 1 &amp; 0 \end{bmatrix}\)</span> has eigenvalues <span class="math inline">\(\pm i\)</span>.</li>
</ul></li>
</ol>
</section>
<section id="geometric-meaning-of-complex-eigenvalues" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-of-complex-eigenvalues">Geometric Meaning of Complex Eigenvalues</h4>
<ul>
<li>If eigenvalues are real, the transformation scales along real directions.</li>
<li>If eigenvalues are complex, the transformation involves a combination of rotation and scaling.</li>
</ul>
<p>For <span class="math inline">\(\lambda = re^{i\theta}\)</span>:</p>
<ul>
<li><span class="math inline">\(r = |\lambda|\)</span> controls expansion or contraction.</li>
<li><span class="math inline">\(\theta\)</span> controls rotation.</li>
</ul>
<p>So a complex eigenvalue represents a spiral: stretching or shrinking while rotating.</p>
</section>
<section id="example-spiral-dynamics" class="level4">
<h4 class="anchored" data-anchor-id="example-spiral-dynamics">Example: Spiral Dynamics</h4>
<p>Matrix</p>
<p><span class="math display">\[
A = \begin{bmatrix} 0 &amp; -1 \\ 1 &amp; 0 \end{bmatrix}
\]</span></p>
<p>rotates vectors by 90°.</p>
<ul>
<li>Eigenvalues: <span class="math inline">\(\pm i\)</span>.</li>
<li>Magnitude = 1, angle = <span class="math inline">\(\pi/2\)</span>.</li>
<li>Interpretation: every step is a rotation of 90°, with no scaling.</li>
</ul>
<p>If we change to</p>
<p><span class="math display">\[
B = \begin{bmatrix} 0.8 &amp; -0.6 \\ 0.6 &amp; 0.8 \end{bmatrix},
\]</span></p>
<p>the eigenvalues are complex with modulus &lt; 1.</p>
<ul>
<li>Interpretation: rotation combined with shrinking → spiraling toward the origin.</li>
</ul>
</section>
<section id="applications-28" class="level4">
<h4 class="anchored" data-anchor-id="applications-28">Applications</h4>
<ol type="1">
<li>Differential Equations: Complex eigenvalues produce oscillatory solutions with sine and cosine terms.</li>
<li>Physics: Vibrations and wave phenomena rely on complex eigenvalues to model periodic behavior.</li>
<li>Control Systems: Stability requires checking magnitudes of eigenvalues in the complex plane.</li>
<li>Computer Graphics: Rotations and spiral motions are naturally described by complex spectra.</li>
<li>Signal Processing: Fourier transforms rely on complex eigenstructures of convolution operators.</li>
</ol>
</section>
<section id="why-it-matters-62" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-62">Why It Matters</h4>
<ul>
<li>Real eigenvalues describe pure stretching or compression.</li>
<li>Complex eigenvalues describe combined rotation and scaling.</li>
<li>Together, they provide a complete picture of matrix behavior in both real and complex spaces.</li>
<li>Without considering complex eigenvalues, we miss entire classes of transformations, like rotation and oscillation.</li>
</ul>
</section>
<section id="try-it-yourself-65" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-65">Try It Yourself</h4>
<ol type="1">
<li>Find eigenvalues of <span class="math inline">\(\begin{bmatrix} 0 &amp; -1 \\ 1 &amp; 0 \end{bmatrix}\)</span>. Interpret geometrically.</li>
<li>For rotation by 45°, find eigenvalues of <span class="math inline">\(\begin{bmatrix} \cos\frac{\pi}{4} &amp; -\sin\frac{\pi}{4} \\ \sin\frac{\pi}{4} &amp; \cos\frac{\pi}{4} \end{bmatrix}\)</span>. Show that they are <span class="math inline">\(e^{\pm i\pi/4}\)</span>.</li>
<li>Check eigenvalues of <span class="math inline">\(\begin{bmatrix} 2 &amp; -5 \\ 1 &amp; -2 \end{bmatrix}\)</span>. Are they real or complex?</li>
<li>Challenge: Prove that real polynomials of odd degree always have at least one real root. Connect this to eigenvalues of odd-dimensional real matrices.</li>
</ol>
<p>Complex spectra extend our understanding of linear algebra into the full richness of oscillations, rotations, and spirals, where numbers alone are not enough-geometry and complex analysis merge to reveal the truth.</p>
</section>
</section>
<section id="defective-matrices-and-jordan-form-a-glimpse" class="level3">
<h3 class="anchored" data-anchor-id="defective-matrices-and-jordan-form-a-glimpse">67. Defective Matrices and Jordan Form (a Glimpse)</h3>
<p>Not every matrix can be simplified all the way into a diagonal form. Some matrices, while having repeated eigenvalues, do not have enough independent eigenvectors to span the entire space. These are called defective matrices. Understanding them requires introducing the Jordan canonical form, a generalization of diagonalization that handles these tricky cases.</p>
<section id="defective-matrices" class="level4">
<h4 class="anchored" data-anchor-id="defective-matrices">Defective Matrices</h4>
<p>A square matrix <span class="math inline">\(A \in \mathbb{R}^{n \times n}\)</span> is called defective if:</p>
<ul>
<li>It has an eigenvalue <span class="math inline">\(\lambda\)</span> with algebraic multiplicity (AM) strictly larger than its geometric multiplicity (GM).</li>
<li>Equivalently, <span class="math inline">\(A\)</span> does not have enough linearly independent eigenvectors to form a full basis of <span class="math inline">\(\mathbb{R}^n\)</span>.</li>
</ul>
<p>Example:</p>
<p><span class="math display">\[
A = \begin{bmatrix} 2 &amp; 1 \\ 0 &amp; 2 \end{bmatrix}.
\]</span></p>
<ul>
<li><p>Characteristic polynomial: <span class="math inline">\((\lambda - 2)^2\)</span>, so AM = 2.</p></li>
<li><p>Solving <span class="math inline">\((A - 2I)v = 0\)</span>:</p>
<p><span class="math display">\[
\begin{bmatrix} 0 &amp; 1 \\ 0 &amp; 0 \end{bmatrix}v = 0 \quad \Rightarrow \quad v = \begin{bmatrix} 1 \\ 0 \end{bmatrix}.
\]</span></p>
<p>Only one independent eigenvector → GM = 1.</p></li>
<li><p>Since GM &lt; AM, this matrix is defective.</p></li>
</ul>
<p>Defective matrices cannot be diagonalized.</p>
</section>
<section id="why-defective-matrices-exist" class="level4">
<h4 class="anchored" data-anchor-id="why-defective-matrices-exist">Why Defective Matrices Exist</h4>
<p>Diagonalization requires one independent eigenvector per eigenvalue copy. But sometimes the matrix “collapses” those directions together, producing fewer eigenvectors than expected.</p>
<ul>
<li>Think of it like having multiple musical notes written in the score (AM), but fewer instruments available to play them (GM).</li>
<li>The matrix “wants” more independent directions, but the geometry of its null spaces prevents that.</li>
</ul>
</section>
<section id="jordan-canonical-form-intuition" class="level4">
<h4 class="anchored" data-anchor-id="jordan-canonical-form-intuition">Jordan Canonical Form (Intuition)</h4>
<p>While defective matrices cannot be diagonalized, they can still be put into a nearly diagonal form called the Jordan canonical form (JCF):</p>
<p><span class="math display">\[
J = P^{-1} A P,
\]</span></p>
<p>where <span class="math inline">\(J\)</span> consists of Jordan blocks:</p>
<p><span class="math display">\[
J_k(\lambda) = \begin{bmatrix}
\lambda &amp; 1 &amp; 0 &amp; \cdots &amp; 0 \\
0 &amp; \lambda &amp; 1 &amp; \cdots &amp; 0 \\
0 &amp; 0 &amp; \lambda &amp; \cdots &amp; 0 \\
\vdots &amp; \vdots &amp; \vdots &amp; \ddots &amp; 1 \\
0 &amp; 0 &amp; 0 &amp; \cdots &amp; \lambda
\end{bmatrix}.
\]</span></p>
<p>Each block corresponds to one eigenvalue <span class="math inline">\(\lambda\)</span>, with 1s on the superdiagonal indicating the lack of independent eigenvectors.</p>
<ul>
<li>If every block is <span class="math inline">\(1 \times 1\)</span>, the matrix is diagonalizable.</li>
<li>If larger blocks appear, the matrix is defective.</li>
</ul>
</section>
<section id="example-jordan-block-of-size-2" class="level4">
<h4 class="anchored" data-anchor-id="example-jordan-block-of-size-2">Example: Jordan Block of Size 2</h4>
<p>The earlier defective example</p>
<p><span class="math display">\[
A = \begin{bmatrix} 2 &amp; 1 \\ 0 &amp; 2 \end{bmatrix}
\]</span></p>
<p>has Jordan form</p>
<p><span class="math display">\[
J = \begin{bmatrix} 2 &amp; 1 \\ 0 &amp; 2 \end{bmatrix}.
\]</span></p>
<p>Notice it is already in Jordan form: one block of size 2 for eigenvalue 2.</p>
</section>
<section id="powers-of-jordan-blocks" class="level4">
<h4 class="anchored" data-anchor-id="powers-of-jordan-blocks">Powers of Jordan Blocks</h4>
<p>A key property is how powers behave. For</p>
<p><span class="math display">\[
J = \begin{bmatrix} \lambda &amp; 1 \\ 0 &amp; \lambda \end{bmatrix},
\]</span></p>
<p><span class="math display">\[
J^k = \begin{bmatrix} \lambda^k &amp; k\lambda^{k-1} \\ 0 &amp; \lambda^k \end{bmatrix}.
\]</span></p>
<ul>
<li>Unlike diagonal matrices, extra polynomial terms in <span class="math inline">\(k\)</span> appear.</li>
<li>This explains why defective matrices produce behavior like growth proportional to <span class="math inline">\(k \lambda^{k-1}\)</span>.</li>
</ul>
</section>
<section id="geometric-meaning-14" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-14">Geometric Meaning</h4>
<ul>
<li>Eigenvectors describe invariant lines.</li>
<li>When there aren’t enough eigenvectors, Jordan form encodes chains of generalized eigenvectors.</li>
<li>Each chain captures how the matrix transforms vectors slightly off the invariant line, nudging them along directions linked together by the Jordan block.</li>
</ul>
<p>So while a diagonalizable matrix decomposes space into neat independent directions, a defective matrix entangles some directions together, forcing them into chains.</p>
</section>
<section id="applications-29" class="level4">
<h4 class="anchored" data-anchor-id="applications-29">Applications</h4>
<ol type="1">
<li>Differential Equations: Jordan blocks determine the appearance of extra polynomial factors (like <span class="math inline">\(t e^{\lambda t}\)</span>) in solutions.</li>
<li>Markov Chains: Non-diagonalizable transition matrices produce slower convergence to steady states.</li>
<li>Numerical Analysis: Algorithms may fail or slow down if the system matrix is defective.</li>
<li>Control Theory: Stability depends not just on eigenvalues, but on whether the matrix is diagonalizable.</li>
<li>Quantum Mechanics: Degenerate eigenvalues require Jordan analysis to fully describe states.</li>
</ol>
</section>
<section id="why-it-matters-63" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-63">Why It Matters</h4>
<ul>
<li>Diagonalization is not always possible, and defective matrices are the exceptions.</li>
<li>Jordan form is the universal fallback: every square matrix has one, and it generalizes diagonalization.</li>
<li>It introduces generalized eigenvectors, which extend the reach of spectral theory.</li>
</ul>
</section>
<section id="try-it-yourself-66" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-66">Try It Yourself</h4>
<ol type="1">
<li><p>Verify that <span class="math inline">\(\begin{bmatrix} 3 &amp; 1 \\ 0 &amp; 3 \end{bmatrix}\)</span> is defective. Find its Jordan form.</p></li>
<li><p>Show that for a Jordan block of size 3,</p>
<p><span class="math display">\[
J^k = \lambda^k I + k \lambda^{k-1} N + \frac{k(k-1)}{2}\lambda^{k-2} N^2,
\]</span></p>
<p>where <span class="math inline">\(N\)</span> is the nilpotent part (matrix with 1s above diagonal).</p></li>
<li><p>Compare the behavior of <span class="math inline">\(A^k\)</span> for a diagonalizable vs.&nbsp;a defective matrix with the same eigenvalues.</p></li>
<li><p>Challenge: Prove that every square matrix has a Jordan form over the complex numbers.</p></li>
</ol>
<p>Defective matrices and Jordan form show us that even when eigenvectors are “insufficient,” we can still impose structure, capturing how linear transformations behave in their most fundamental building blocks.</p>
</section>
</section>
<section id="stability-and-spectral-radius" class="level3">
<h3 class="anchored" data-anchor-id="stability-and-spectral-radius">68. Stability and Spectral Radius</h3>
<p>When a matrix is applied repeatedly-through iteration, recursion, or dynamical systems-its long-term behavior is governed not by individual entries, but by its eigenvalues. The key measure here is the spectral radius, which tells us whether repeated applications lead to convergence, oscillation, or divergence.</p>
<section id="the-spectral-radius" class="level4">
<h4 class="anchored" data-anchor-id="the-spectral-radius">The Spectral Radius</h4>
<p>The spectral radius of a matrix <span class="math inline">\(A\)</span> is defined as</p>
<p><span class="math display">\[
\rho(A) = \max \{ |\lambda| : \lambda \text{ is an eigenvalue of } A \}.
\]</span></p>
<ul>
<li>It is the largest absolute value among all eigenvalues.</li>
<li>If <span class="math inline">\(|\lambda| &gt; 1\)</span>, the eigenvalue leads to exponential growth along its eigenvector.</li>
<li>If <span class="math inline">\(|\lambda| &lt; 1\)</span>, it leads to exponential decay.</li>
<li>If <span class="math inline">\(|\lambda| = 1\)</span>, behavior depends on whether the eigenvalue is simple or defective.</li>
</ul>
</section>
<section id="stability-in-iterative-systems" class="level4">
<h4 class="anchored" data-anchor-id="stability-in-iterative-systems">Stability in Iterative Systems</h4>
<p>Consider a recursive process:</p>
<p><span class="math display">\[
x_{k+1} = A x_k.
\]</span></p>
<ul>
<li>If <span class="math inline">\(\rho(A) &lt; 1\)</span>, then <span class="math inline">\(A^k \to 0\)</span> as <span class="math inline">\(k \to \infty\)</span>. All trajectories converge to the origin.</li>
<li>If <span class="math inline">\(\rho(A) &gt; 1\)</span>, then <span class="math inline">\(A^k\)</span> grows without bound along the dominant eigenvector.</li>
<li>If <span class="math inline">\(\rho(A) = 1\)</span>, trajectories neither vanish nor diverge but may oscillate or stagnate.</li>
</ul>
</section>
<section id="example-convergence-with-small-spectral-radius" class="level4">
<h4 class="anchored" data-anchor-id="example-convergence-with-small-spectral-radius">Example: Convergence with Small Spectral Radius</h4>
<p><span class="math display">\[
A = \begin{bmatrix} 0.5 &amp; 0 \\ 0 &amp; 0.8 \end{bmatrix}.
\]</span></p>
<ul>
<li>Eigenvalues: <span class="math inline">\(0.5, 0.8\)</span>.</li>
<li><span class="math inline">\(\rho(A) = 0.8 &lt; 1\)</span>.</li>
<li>Powers <span class="math inline">\(A^k\)</span> shrink vectors to zero → stable system.</li>
</ul>
</section>
<section id="example-divergence-with-large-spectral-radius" class="level4">
<h4 class="anchored" data-anchor-id="example-divergence-with-large-spectral-radius">Example: Divergence with Large Spectral Radius</h4>
<p><span class="math display">\[
B = \begin{bmatrix} 2 &amp; 0 \\ 0 &amp; 0.5 \end{bmatrix}.
\]</span></p>
<ul>
<li>Eigenvalues: <span class="math inline">\(2, 0.5\)</span>.</li>
<li><span class="math inline">\(\rho(B) = 2 &gt; 1\)</span>.</li>
<li>Powers <span class="math inline">\(B^k\)</span> explode along the eigenvector <span class="math inline">\((1,0)\)</span>.</li>
</ul>
</section>
<section id="example-oscillation-with-complex-eigenvalues" class="level4">
<h4 class="anchored" data-anchor-id="example-oscillation-with-complex-eigenvalues">Example: Oscillation with Complex Eigenvalues</h4>
<p><span class="math display">\[
C = \begin{bmatrix} 0 &amp; -1 \\ 1 &amp; 0 \end{bmatrix}.
\]</span></p>
<ul>
<li>Eigenvalues: <span class="math inline">\(\pm i\)</span>, both with modulus 1.</li>
<li><span class="math inline">\(\rho(C) = 1\)</span>.</li>
<li>System is neutrally stable: vectors rotate forever without shrinking or growing.</li>
</ul>
</section>
<section id="beyond-simple-stability-defective-cases" class="level4">
<h4 class="anchored" data-anchor-id="beyond-simple-stability-defective-cases">Beyond Simple Stability: Defective Cases</h4>
<p>If a matrix has eigenvalues with <span class="math inline">\(|\lambda|=1\)</span> and is defective, extra polynomial terms in <span class="math inline">\(k\)</span> appear in <span class="math inline">\(A^k\)</span>, leading to slow divergence even though <span class="math inline">\(\rho(A)=1\)</span>.</p>
<p>Example:</p>
<p><span class="math display">\[
D = \begin{bmatrix} 1 &amp; 1 \\ 0 &amp; 1 \end{bmatrix}.
\]</span></p>
<ul>
<li><p>Eigenvalue: <span class="math inline">\(\lambda=1\)</span> (AM=2, GM=1).</p></li>
<li><p><span class="math inline">\(\rho(D)=1\)</span>.</p></li>
<li><p>Powers grow linearly with <span class="math inline">\(k\)</span>:</p>
<p><span class="math display">\[
D^k = \begin{bmatrix} 1 &amp; k \\ 0 &amp; 1 \end{bmatrix}.
\]</span></p></li>
<li><p>System is unstable, despite spectral radius equal to 1.</p></li>
</ul>
</section>
<section id="geometric-meaning-15" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-15">Geometric Meaning</h4>
<p>The spectral radius measures the dominant mode of a transformation:</p>
<ul>
<li>Imagine stretching and rotating a rubber sheet. After many repetitions, the sheet aligns with the direction corresponding to the largest eigenvalue.</li>
<li>If the stretching is less than 1, everything shrinks.</li>
<li>If greater than 1, everything expands.</li>
<li>If exactly 1, the system is balanced on the edge of stability.</li>
</ul>
</section>
<section id="applications-30" class="level4">
<h4 class="anchored" data-anchor-id="applications-30">Applications</h4>
<ol type="1">
<li>Numerical Methods: Convergence of iterative solvers (e.g., Jacobi, Gauss–Seidel) depends on spectral radius &lt; 1.</li>
<li>Markov Chains: Long-term distributions exist if the largest eigenvalue = 1 and others &lt; 1 in magnitude.</li>
<li>Control Theory: System stability is judged by eigenvalues inside the unit circle (<span class="math inline">\(|\lambda| &lt; 1\)</span>).</li>
<li>Economics: Input-output models remain bounded only if spectral radius &lt; 1.</li>
<li>Epidemiology: Basic reproduction number <span class="math inline">\(R_0\)</span> is essentially the spectral radius of a next-generation matrix.</li>
</ol>
</section>
<section id="why-it-matters-64" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-64">Why It Matters</h4>
<ul>
<li>The spectral radius condenses the entire spectrum of a matrix into a single stability criterion.</li>
<li>It predicts the fate of iterative processes, from financial growth to disease spread.</li>
<li>It draws a sharp boundary between decay, balance, and explosion in linear systems.</li>
</ul>
</section>
<section id="try-it-yourself-67" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-67">Try It Yourself</h4>
<ol type="1">
<li><p>Compute the spectral radius of <span class="math inline">\(\begin{bmatrix} 0.6 &amp; 0.3 \\ 0.1 &amp; 0.8 \end{bmatrix}\)</span>. Does the system converge?</p></li>
<li><p>Show that for any matrix norm <span class="math inline">\(\|\cdot\|\)</span>,</p>
<p><span class="math display">\[
\rho(A) \leq \|A\|.
\]</span></p>
<p>(Hint: use Gelfand’s formula.)</p></li>
<li><p>For <span class="math inline">\(\begin{bmatrix} 1 &amp; 1 \\ 0 &amp; 1 \end{bmatrix}\)</span>, explain why it diverges even though <span class="math inline">\(\rho=1\)</span>.</p></li>
<li><p>Challenge: Prove Gelfand’s formula:</p>
<p><span class="math display">\[
\rho(A) = \lim_{k\to\infty} \|A^k\|^{1/k}.
\]</span></p></li>
</ol>
<p>The spectral radius is the compass of linear dynamics: it points to stability, oscillation, or divergence, guiding us across disciplines wherever repeated transformations shape the future.</p>
</section>
</section>
<section id="markov-chains-and-steady-states" class="level3">
<h3 class="anchored" data-anchor-id="markov-chains-and-steady-states">69. Markov Chains and Steady States</h3>
<p>Markov chains are one of the most direct and beautiful applications of eigenvalues in probability and statistics. They describe systems that evolve step by step, where the next state depends only on the current one, not on the past. The mathematics of steady states-the long-term behavior of such chains-rests firmly on eigenvalues and eigenvectors of the transition matrix.</p>
<section id="transition-matrices" class="level4">
<h4 class="anchored" data-anchor-id="transition-matrices">Transition Matrices</h4>
<p>A Markov chain is defined by a transition matrix <span class="math inline">\(P \in \mathbb{R}^{n \times n}\)</span> with the following properties:</p>
<ol type="1">
<li>All entries are nonnegative: <span class="math inline">\(p_{ij} \geq 0\)</span>.</li>
<li>Each row sums to 1: <span class="math inline">\(\sum_j p_{ij} = 1\)</span>.</li>
</ol>
<p>If the chain is in state <span class="math inline">\(i\)</span> at time <span class="math inline">\(k\)</span>, then <span class="math inline">\(p_{ij}\)</span> is the probability of moving to state <span class="math inline">\(j\)</span> at time <span class="math inline">\(k+1\)</span>.</p>
</section>
<section id="evolution-of-states" class="level4">
<h4 class="anchored" data-anchor-id="evolution-of-states">Evolution of States</h4>
<p>If the probability distribution at time <span class="math inline">\(k\)</span> is a row vector <span class="math inline">\(\pi^{(k)}\)</span>, then</p>
<p><span class="math display">\[
\pi^{(k+1)} = \pi^{(k)} P.
\]</span></p>
<p>After <span class="math inline">\(k\)</span> steps:</p>
<p><span class="math display">\[
\pi^{(k)} = \pi^{(0)} P^k.
\]</span></p>
<p>So understanding the long-term behavior requires analyzing <span class="math inline">\(P^k\)</span>.</p>
</section>
<section id="eigenvalue-structure-of-transition-matrices" class="level4">
<h4 class="anchored" data-anchor-id="eigenvalue-structure-of-transition-matrices">Eigenvalue Structure of Transition Matrices</h4>
<ul>
<li><p>Every transition matrix <span class="math inline">\(P\)</span> has eigenvalue <span class="math inline">\(\lambda = 1\)</span>.</p></li>
<li><p>All other eigenvalues satisfy <span class="math inline">\(|\lambda| \leq 1\)</span>.</p></li>
<li><p>If the chain is irreducible (all states communicate) and aperiodic (no cyclic locking), then:</p>
<ul>
<li><span class="math inline">\(\lambda=1\)</span> is a simple eigenvalue (AM=GM=1).</li>
<li>All other eigenvalues have magnitude strictly less than 1.</li>
</ul></li>
</ul>
<p>This ensures convergence to a unique steady state.</p>
</section>
<section id="steady-states-as-eigenvectors" class="level4">
<h4 class="anchored" data-anchor-id="steady-states-as-eigenvectors">Steady States as Eigenvectors</h4>
<p>A steady state distribution <span class="math inline">\(\pi\)</span> satisfies:</p>
<p><span class="math display">\[
\pi = \pi P.
\]</span></p>
<p>This is equivalent to:</p>
<p><span class="math display">\[
\pi^T \text{ is a right eigenvector of } P^T \text{ with eigenvalue } 1.
\]</span></p>
<ul>
<li>The steady state vector lies in the eigenspace of eigenvalue 1.</li>
<li>Since probabilities must sum to 1, normalization gives a unique steady state.</li>
</ul>
</section>
<section id="example-a-2-state-markov-chain" class="level4">
<h4 class="anchored" data-anchor-id="example-a-2-state-markov-chain">Example: A 2-State Markov Chain</h4>
<p><span class="math display">\[
P = \begin{bmatrix} 0.7 &amp; 0.3 \\ 0.4 &amp; 0.6 \end{bmatrix}.
\]</span></p>
<ul>
<li><p>Eigenvalues: solve <span class="math inline">\(\det(P-\lambda I) = 0\)</span>.</p>
<p><span class="math display">\[
\lambda_1 = 1, \quad \lambda_2 = 0.3.
\]</span></p></li>
<li><p>The steady state is found from <span class="math inline">\(\pi = \pi P\)</span>:</p>
<p><span class="math display">\[
\pi = \bigg(\frac{4}{7}, \frac{3}{7}\bigg).
\]</span></p></li>
<li><p>As <span class="math inline">\(k \to \infty\)</span>, any initial distribution <span class="math inline">\(\pi^{(0)}\)</span> converges to this steady state.</p></li>
</ul>
</section>
<section id="example-random-walk-on-a-graph" class="level4">
<h4 class="anchored" data-anchor-id="example-random-walk-on-a-graph">Example: Random Walk on a Graph</h4>
<p>Take a simple graph: 3 nodes in a line, where each node passes to neighbors equally.</p>
<p>Transition matrix:</p>
<p><span class="math display">\[
P = \begin{bmatrix}
0 &amp; 1 &amp; 0 \\
0.5 &amp; 0 &amp; 0.5 \\
0 &amp; 1 &amp; 0
\end{bmatrix}.
\]</span></p>
<ul>
<li>Eigenvalues: <span class="math inline">\(\{1, 0, -1\}\)</span>.</li>
<li>The steady state corresponds to eigenvalue 1.</li>
<li>After many steps, the distribution converges to <span class="math inline">\((0.25, 0.5, 0.25)\)</span>.</li>
</ul>
</section>
<section id="geometric-meaning-16" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-16">Geometric Meaning</h4>
<ul>
<li>Eigenvalue 1: the fixed “direction” of probabilities that does not change under transitions.</li>
<li>Eigenvalues &lt; 1 in magnitude: transient modes that vanish as <span class="math inline">\(k \to \infty\)</span>.</li>
<li>The dominant eigenvector (steady state) is like the “center of gravity” of the system.</li>
</ul>
<p>So powers of <span class="math inline">\(P\)</span> filter out all but the eigenvector of eigenvalue 1.</p>
</section>
<section id="applications-31" class="level4">
<h4 class="anchored" data-anchor-id="applications-31">Applications</h4>
<ol type="1">
<li>Google PageRank: Steady state eigenvectors rank webpages.</li>
<li>Economics: Input-output models evolve like Markov chains.</li>
<li>Epidemiology: Spread of diseases can be modeled as Markov processes.</li>
<li>Machine Learning: Hidden Markov models (HMMs) underpin speech recognition and bioinformatics.</li>
<li>Queuing Theory: Customer arrivals and service evolve according to Markov dynamics.</li>
</ol>
</section>
<section id="why-it-matters-65" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-65">Why It Matters</h4>
<ul>
<li>The concept of steady states shows how randomness can lead to predictability.</li>
<li>Eigenvalues explain why convergence happens, and at what rate.</li>
<li>The link between linear algebra and probability provides one of the clearest real-world uses of eigenvectors.</li>
</ul>
</section>
<section id="try-it-yourself-68" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-68">Try It Yourself</h4>
<ol type="1">
<li><p>For</p>
<p><span class="math display">\[
P = \begin{bmatrix} 0.9 &amp; 0.1 \\ 0.5 &amp; 0.5 \end{bmatrix},
\]</span></p>
<p>compute its eigenvalues and steady state.</p></li>
<li><p>Show that for any transition matrix, the largest eigenvalue is always 1.</p></li>
<li><p>Prove that if a chain is irreducible and aperiodic, the steady state is unique.</p></li>
<li><p>Challenge: Construct a 3-state transition matrix with a cycle (periodic) and show why it doesn’t converge to a steady distribution until perturbed.</p></li>
</ol>
<p>Markov chains and steady states are the meeting point of probability and linear algebra: randomness, when multiplied many times, is tamed by the calm persistence of eigenvalue 1.</p>
</section>
</section>
<section id="linear-differential-systems" class="level3">
<h3 class="anchored" data-anchor-id="linear-differential-systems">70. Linear Differential Systems</h3>
<p>Many natural and engineered processes evolve continuously over time. When these processes can be expressed as linear relationships, they lead to systems of linear differential equations. The analysis of such systems relies almost entirely on eigenvalues and eigenvectors, which determine the behavior of solutions: whether they oscillate, decay, grow, or stabilize.</p>
<section id="the-general-setup" class="level4">
<h4 class="anchored" data-anchor-id="the-general-setup">The General Setup</h4>
<p>Consider a system of first-order linear differential equations:</p>
<p><span class="math display">\[
\frac{d}{dt}x(t) = A x(t),
\]</span></p>
<p>where:</p>
<ul>
<li><span class="math inline">\(x(t) \in \mathbb{R}^n\)</span> is the state vector at time <span class="math inline">\(t\)</span>.</li>
<li><span class="math inline">\(A \in \mathbb{R}^{n \times n}\)</span> is a constant coefficient matrix.</li>
</ul>
<p>The task is to solve for <span class="math inline">\(x(t)\)</span>, given an initial state <span class="math inline">\(x(0)\)</span>.</p>
</section>
<section id="the-matrix-exponential" class="level4">
<h4 class="anchored" data-anchor-id="the-matrix-exponential">The Matrix Exponential</h4>
<p>The formal solution is:</p>
<p><span class="math display">\[
x(t) = e^{At} x(0),
\]</span></p>
<p>where <span class="math inline">\(e^{At}\)</span> is the matrix exponential defined as:</p>
<p><span class="math display">\[
e^{At} = I + At + \frac{(At)^2}{2!} + \frac{(At)^3}{3!} + \cdots.
\]</span></p>
<p>But how do we compute <span class="math inline">\(e^{At}\)</span> in practice? The answer comes from diagonalization and Jordan form.</p>
</section>
<section id="case-1-diagonalizable-matrices" class="level4">
<h4 class="anchored" data-anchor-id="case-1-diagonalizable-matrices">Case 1: Diagonalizable Matrices</h4>
<p>If <span class="math inline">\(A\)</span> is diagonalizable:</p>
<p><span class="math display">\[
A = P D P^{-1}, \quad D = \text{diag}(\lambda_1, \ldots, \lambda_n).
\]</span></p>
<p>Then:</p>
<p><span class="math display">\[
e^{At} = P e^{Dt} P^{-1}, \quad e^{Dt} = \text{diag}(e^{\lambda_1 t}, \ldots, e^{\lambda_n t}).
\]</span></p>
<p>Thus the solution is:</p>
<p><span class="math display">\[
x(t) = P \begin{bmatrix} e^{\lambda_1 t} &amp; &amp; \\ &amp; \ddots &amp; \\ &amp; &amp; e^{\lambda_n t} \end{bmatrix} P^{-1} x(0).
\]</span></p>
<p>Each eigenvalue <span class="math inline">\(\lambda_i\)</span> dictates the time behavior along its eigenvector direction.</p>
</section>
<section id="case-2-non-diagonalizable-matrices" class="level4">
<h4 class="anchored" data-anchor-id="case-2-non-diagonalizable-matrices">Case 2: Non-Diagonalizable Matrices</h4>
<p>If <span class="math inline">\(A\)</span> is defective, use its Jordan form <span class="math inline">\(J = P^{-1}AP\)</span>:</p>
<p><span class="math display">\[
e^{At} = P e^{Jt} P^{-1}.
\]</span></p>
<p>For a Jordan block of size 2:</p>
<p><span class="math display">\[
J = \begin{bmatrix} \lambda &amp; 1 \\ 0 &amp; \lambda \end{bmatrix}, \quad
e^{Jt} = e^{\lambda t} \begin{bmatrix} 1 &amp; t \\ 0 &amp; 1 \end{bmatrix}.
\]</span></p>
<p>Polynomial terms in <span class="math inline">\(t\)</span> appear, multiplying the exponential part. This explains why repeated eigenvalues with insufficient eigenvectors yield solutions with extra polynomial factors.</p>
</section>
<section id="real-vs.-complex-eigenvalues" class="level4">
<h4 class="anchored" data-anchor-id="real-vs.-complex-eigenvalues">Real vs.&nbsp;Complex Eigenvalues</h4>
<ul>
<li><p>Real eigenvalues: solutions grow or decay exponentially along eigenvector directions.</p>
<ul>
<li>If <span class="math inline">\(\lambda &lt; 0\)</span>: exponential decay → stability.</li>
<li>If <span class="math inline">\(\lambda &gt; 0\)</span>: exponential growth → instability.</li>
</ul></li>
<li><p>Complex eigenvalues: <span class="math inline">\(\lambda = a \pm bi\)</span>. Solutions involve oscillations:</p>
<p><span class="math display">\[
e^{(a+bi)t} = e^{at}(\cos(bt) + i \sin(bt)).
\]</span></p>
<ul>
<li>If <span class="math inline">\(a &lt; 0\)</span>: decaying oscillations.</li>
<li>If <span class="math inline">\(a &gt; 0\)</span>: growing oscillations.</li>
<li>If <span class="math inline">\(a = 0\)</span>: pure oscillations, neutrally stable.</li>
</ul></li>
</ul>
</section>
<section id="example-1-real-eigenvalues" class="level4">
<h4 class="anchored" data-anchor-id="example-1-real-eigenvalues">Example 1: Real Eigenvalues</h4>
<p><span class="math display">\[
A = \begin{bmatrix} -2 &amp; 0 \\ 0 &amp; -3 \end{bmatrix}.
\]</span></p>
<p>Eigenvalues: <span class="math inline">\(-2, -3\)</span>. Solution:</p>
<p><span class="math display">\[
x(t) = \begin{bmatrix} c_1 e^{-2t} \\ c_2 e^{-3t} \end{bmatrix}.
\]</span></p>
<p>Both terms decay → stable equilibrium at the origin.</p>
</section>
<section id="example-2-complex-eigenvalues" class="level4">
<h4 class="anchored" data-anchor-id="example-2-complex-eigenvalues">Example 2: Complex Eigenvalues</h4>
<p><span class="math display">\[
A = \begin{bmatrix} 0 &amp; -1 \\ 1 &amp; 0 \end{bmatrix}.
\]</span></p>
<p>Eigenvalues: <span class="math inline">\(\pm i\)</span>. Solution:</p>
<p><span class="math display">\[
x(t) = c_1 \begin{bmatrix} \cos t \\ \sin t \end{bmatrix} + c_2 \begin{bmatrix} -\sin t \\ \cos t \end{bmatrix}.
\]</span></p>
<p>Pure oscillation → circular motion around the origin.</p>
</section>
<section id="example-3-mixed-stability" class="level4">
<h4 class="anchored" data-anchor-id="example-3-mixed-stability">Example 3: Mixed Stability</h4>
<p><span class="math display">\[
A = \begin{bmatrix} 1 &amp; 0 \\ 0 &amp; -2 \end{bmatrix}.
\]</span></p>
<p>Eigenvalues: <span class="math inline">\(1, -2\)</span>. Solution:</p>
<p><span class="math display">\[
x(t) = \begin{bmatrix} c_1 e^t \\ c_2 e^{-2t} \end{bmatrix}.
\]</span></p>
<p>One direction grows, one decays → unstable overall, since divergence in one direction dominates.</p>
</section>
<section id="geometric-meaning-17" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-17">Geometric Meaning</h4>
<ul>
<li>The eigenvectors form the “axes of flow” of the system.</li>
<li>The eigenvalues determine whether the flow along those axes spirals, grows, or shrinks.</li>
<li>The phase portrait of the system-trajectories in the plane-is shaped by this interplay.</li>
</ul>
<p>For example:</p>
<ul>
<li>Negative eigenvalues → trajectories funnel into the origin.</li>
<li>Positive eigenvalues → trajectories repel outward.</li>
<li>Complex eigenvalues → spirals or circles.</li>
</ul>
</section>
<section id="applications-32" class="level4">
<h4 class="anchored" data-anchor-id="applications-32">Applications</h4>
<ol type="1">
<li>Control theory: Stability analysis of systems requires eigenvalue placement in the left-half plane.</li>
<li>Physics: Vibrations, quantum oscillations, and decay processes all follow eigenvalue rules.</li>
<li>Biology: Population models evolve according to linear differential equations.</li>
<li>Economics: Linear models of markets converge or diverge depending on eigenvalues.</li>
<li>Neuroscience: Neural firing dynamics can be modeled as linear ODE systems.</li>
</ol>
</section>
<section id="why-it-matters-66" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-66">Why It Matters</h4>
<ul>
<li>Linear differential systems bridge linear algebra with real-world dynamics.</li>
<li>Eigenvalues determine not just numbers, but behaviors over time: growth, decay, oscillation, or equilibrium.</li>
<li>They provide the foundation for analyzing nonlinear systems, which are often studied by linearizing around equilibrium points.</li>
</ul>
</section>
<section id="try-it-yourself-69" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-69">Try It Yourself</h4>
<ol type="1">
<li>Solve <span class="math inline">\(\frac{dx}{dt} = \begin{bmatrix} -1 &amp; 2 \\ -2 &amp; -1 \end{bmatrix}x\)</span>. Interpret the solution.</li>
<li>For <span class="math inline">\(A = \begin{bmatrix} 0 &amp; -2 \\ 2 &amp; 0 \end{bmatrix}\)</span>, compute eigenvalues and describe the motion.</li>
<li>Verify that <span class="math inline">\(e^{At} = P e^{Dt} P^{-1}\)</span> works when <span class="math inline">\(A\)</span> is diagonalizable.</li>
<li>Challenge: Show that if all eigenvalues of <span class="math inline">\(A\)</span> have negative real parts, then <span class="math inline">\(\lim_{t \to \infty} x(t) = 0\)</span> for any initial condition.</li>
</ol>
<p>Linear differential systems show how eigenvalues control the flow of time itself in models. They explain why some processes die out, others oscillate, and others grow without bound-providing the mathematical skeleton behind countless real-world phenomena.</p>
</section>
<section id="closing-6" class="level4">
<h4 class="anchored" data-anchor-id="closing-6">Closing</h4>
<pre><code>Spectra guide the flow,
growth and decay intertwining,
future sings through roots.</code></pre>
</section>
</section>
</section>
<section id="chapter-8.-orthogonality-least-squars-and-qr" class="level2">
<h2 class="anchored" data-anchor-id="chapter-8.-orthogonality-least-squars-and-qr">Chapter 8. Orthogonality, least squars, and QR</h2>
<section id="opening-6" class="level4">
<h4 class="anchored" data-anchor-id="opening-6">Opening</h4>
<pre><code>Perpendiculars,
meeting without crossing paths,
balance in silence.</code></pre>
</section>
<section id="inner-products-beyond-dot-product" class="level3">
<h3 class="anchored" data-anchor-id="inner-products-beyond-dot-product">71. Inner Products Beyond Dot Product</h3>
<p>The dot product is the first inner product most students encounter. In <span class="math inline">\(\mathbb{R}^n\)</span>, it is defined as</p>
<p><span class="math display">\[
\langle x, y \rangle = x \cdot y = \sum_{i=1}^n x_i y_i,
\]</span></p>
<p>and it provides a way to measure length, angle, and orthogonality. But the dot product is just one special case of a much broader concept. Inner products generalize the dot product, extending its geometric intuition to more abstract vector spaces.</p>
<section id="definition-of-an-inner-product" class="level4">
<h4 class="anchored" data-anchor-id="definition-of-an-inner-product">Definition of an Inner Product</h4>
<p>An inner product on a real vector space <span class="math inline">\(V\)</span> is a function</p>
<p><span class="math display">\[
\langle \cdot, \cdot \rangle : V \times V \to \mathbb{R}
\]</span></p>
<p>that satisfies the following axioms for all <span class="math inline">\(x,y,z \in V\)</span> and scalar <span class="math inline">\(\alpha \in \mathbb{R}\)</span>:</p>
<ol type="1">
<li>Positivity: <span class="math inline">\(\langle x, x \rangle \geq 0\)</span>, and <span class="math inline">\(\langle x, x \rangle = 0 \iff x=0\)</span>.</li>
<li>Symmetry: <span class="math inline">\(\langle x, y \rangle = \langle y, x \rangle\)</span>.</li>
<li>Linearity in the first argument: <span class="math inline">\(\langle \alpha x + y, z \rangle = \alpha \langle x, z \rangle + \langle y, z \rangle\)</span>.</li>
</ol>
<p>In complex vector spaces, the symmetry condition changes to conjugate symmetry: <span class="math inline">\(\langle x, y \rangle = \overline{\langle y, x \rangle}\)</span>.</p>
</section>
<section id="norms-and-angles-from-inner-products" class="level4">
<h4 class="anchored" data-anchor-id="norms-and-angles-from-inner-products">Norms and Angles from Inner Products</h4>
<p>Once an inner product is defined, we immediately get:</p>
<ul>
<li><p>Norm (length): <span class="math inline">\(\|x\| = \sqrt{\langle x, x \rangle}\)</span>.</p></li>
<li><p>Distance: <span class="math inline">\(d(x,y) = \|x-y\|\)</span>.</p></li>
<li><p>Angle between vectors: <span class="math inline">\(\cos \theta = \frac{\langle x, y \rangle}{\|x\|\|y\|}\)</span>.</p></li>
</ul>
<p>Thus, inner products generalize the familiar geometry of <span class="math inline">\(\mathbb{R}^n\)</span> to broader contexts.</p>
</section>
<section id="examples-beyond-the-dot-product" class="level4">
<h4 class="anchored" data-anchor-id="examples-beyond-the-dot-product">Examples Beyond the Dot Product</h4>
<ol type="1">
<li><p>Weighted Inner Product (in <span class="math inline">\(\mathbb{R}^n\)</span>):</p>
<p><span class="math display">\[
\langle x, y \rangle_W = x^T W y,
\]</span></p>
<p>where <span class="math inline">\(W\)</span> is a symmetric positive definite matrix.</p>
<ul>
<li>Here, lengths and angles depend on the weights encoded in <span class="math inline">\(W\)</span>.</li>
<li>Useful when some dimensions are more important than others (e.g., weighted least squares).</li>
</ul></li>
<li><p>Function Spaces (continuous inner product): On <span class="math inline">\(V = C[a,b]\)</span>, the space of continuous functions on <span class="math inline">\([a,b]\)</span>:</p>
<p><span class="math display">\[
\langle f, g \rangle = \int_a^b f(t) g(t) \, dt.
\]</span></p>
<ul>
<li>Length: <span class="math inline">\(\|f\| = \sqrt{\int_a^b f(t)^2 dt}\)</span>.</li>
<li>Orthogonality: <span class="math inline">\(f\)</span> and <span class="math inline">\(g\)</span> are orthogonal if their integral product is zero.</li>
<li>This inner product underpins Fourier series.</li>
</ul></li>
<li><p>Complex Inner Product (in <span class="math inline">\(\mathbb{C}^n\)</span>):</p>
<p><span class="math display">\[
\langle x, y \rangle = \sum_{i=1}^n x_i \overline{y_i}.
\]</span></p>
<ul>
<li>Conjugation ensures positivity.</li>
<li>Critical for quantum mechanics, where states are vectors in complex Hilbert spaces.</li>
</ul></li>
<li><p>Polynomial Spaces: For polynomials on <span class="math inline">\([-1,1]\)</span>:</p>
<p><span class="math display">\[
\langle p, q \rangle = \int_{-1}^1 p(x) q(x) \, dx.
\]</span></p>
<ul>
<li>Leads to orthogonal polynomials (Legendre, Chebyshev), fundamental in approximation theory.</li>
</ul></li>
</ol>
</section>
<section id="geometric-interpretation-15" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-15">Geometric Interpretation</h4>
<ul>
<li>Inner products reshape geometry. Instead of measuring lengths and angles with the Euclidean metric, we measure them with the metric induced by the chosen inner product.</li>
<li>Different inner products create different geometries on the same vector space.</li>
</ul>
<p>Example: A weighted inner product distorts circles into ellipses, changing which vectors count as “orthogonal.”</p>
</section>
<section id="applications-33" class="level4">
<h4 class="anchored" data-anchor-id="applications-33">Applications</h4>
<ol type="1">
<li>Signal Processing: Correlation between signals is an inner product. Orthogonality means two signals carry independent information.</li>
<li>Fourier Analysis: Fourier coefficients come from inner products with sine and cosine functions.</li>
<li>Machine Learning: Kernel methods generalize inner products to infinite-dimensional spaces.</li>
<li>Quantum Mechanics: Probabilities are squared magnitudes of complex inner products.</li>
<li>Optimization: Weighted least squares problems use weighted inner products.</li>
</ol>
</section>
<section id="why-it-matters-67" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-67">Why It Matters</h4>
<ul>
<li>Inner products generalize geometry to new contexts: weighted spaces, functions, polynomials, quantum states.</li>
<li>They provide the foundation for defining orthogonality, projections, and orthonormal bases in spaces far beyond <span class="math inline">\(\mathbb{R}^n\)</span>.</li>
<li>They unify ideas across pure mathematics, physics, engineering, and computer science.</li>
</ul>
</section>
<section id="try-it-yourself-70" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-70">Try It Yourself</h4>
<ol type="1">
<li>Show that the weighted inner product <span class="math inline">\(\langle x, y \rangle_W = x^T W y\)</span> satisfies the inner product axioms if <span class="math inline">\(W\)</span> is positive definite.</li>
<li>Compute <span class="math inline">\(\langle f, g \rangle = \int_0^\pi \sin(t)\cos(t)\, dt\)</span>. Are <span class="math inline">\(f=\sin\)</span> and <span class="math inline">\(g=\cos\)</span> orthogonal?</li>
<li>In <span class="math inline">\(\mathbb{C}^2\)</span>, verify that <span class="math inline">\(\langle (1,i), (i,1) \rangle = 0\)</span>. What does this mean geometrically?</li>
<li>Challenge: Prove that every inner product induces a norm, and that different inner products can lead to different geometries on the same space.</li>
</ol>
<p>The dot product is just the beginning. Inner products provide the language to extend geometry into weighted spaces, continuous functions, and infinite dimensions-transforming how we measure similarity, distance, and structure across mathematics and science.</p>
</section>
</section>
<section id="orthogonality-and-orthonormal-bases" class="level3">
<h3 class="anchored" data-anchor-id="orthogonality-and-orthonormal-bases">72. Orthogonality and Orthonormal Bases</h3>
<p>Orthogonality is one of the most powerful ideas in linear algebra. It generalizes the familiar concept of perpendicularity in Euclidean space to abstract vector spaces equipped with an inner product. When orthogonality is combined with normalization (making vectors have unit length), we obtain orthonormal bases, which simplify computations, clarify geometry, and underpin many algorithms.</p>
<section id="orthogonality-1" class="level4">
<h4 class="anchored" data-anchor-id="orthogonality-1">Orthogonality</h4>
<p>Two vectors <span class="math inline">\(x, y \in V\)</span> are orthogonal if</p>
<p><span class="math display">\[
\langle x, y \rangle = 0.
\]</span></p>
<ul>
<li>In <span class="math inline">\(\mathbb{R}^2\)</span> or <span class="math inline">\(\mathbb{R}^3\)</span>, this means the vectors are perpendicular.</li>
<li>In function spaces, it means the integral of their product is zero.</li>
<li>In signal processing, it means the signals are independent and non-overlapping.</li>
</ul>
<p>Orthogonality captures the idea of “no overlap” or “independence” under the geometry of the inner product.</p>
</section>
<section id="properties-of-orthogonal-vectors" class="level4">
<h4 class="anchored" data-anchor-id="properties-of-orthogonal-vectors">Properties of Orthogonal Vectors</h4>
<ol type="1">
<li>If <span class="math inline">\(x \perp y\)</span>, then <span class="math inline">\(\|x+y\|^2 = \|x\|^2 + \|y\|^2\)</span> (Pythagoras’ theorem generalized).</li>
<li>Orthogonality is symmetric: if <span class="math inline">\(x \perp y\)</span>, then <span class="math inline">\(y \perp x\)</span>.</li>
<li>Any set of mutually orthogonal nonzero vectors is automatically linearly independent.</li>
</ol>
<p>This last property is critical: orthogonality guarantees independence.</p>
</section>
<section id="orthonormal-sets" class="level4">
<h4 class="anchored" data-anchor-id="orthonormal-sets">Orthonormal Sets</h4>
<p>An orthonormal set is a collection of vectors <span class="math inline">\(\{u_1, \dots, u_k\}\)</span> such that</p>
<p><span class="math display">\[
\langle u_i, u_j \rangle = \begin{cases}
1 &amp; \text{if } i=j, \\
0 &amp; \text{if } i \neq j.  
\end{cases}
\]</span></p>
<ul>
<li>Each vector has unit length.</li>
<li>Distinct vectors are mutually orthogonal.</li>
</ul>
<p>This structure makes computations with coordinates as simple as possible.</p>
</section>
<section id="orthonormal-bases" class="level4">
<h4 class="anchored" data-anchor-id="orthonormal-bases">Orthonormal Bases</h4>
<p>A basis <span class="math inline">\(\{u_1, \dots, u_n\}\)</span> for a vector space is orthonormal if it is orthonormal as a set.</p>
<ul>
<li><p>Any vector <span class="math inline">\(x \in V\)</span> can be written as</p>
<p><span class="math display">\[
x = \sum_{i=1}^n \langle x, u_i \rangle u_i.
\]</span></p></li>
<li><p>The coefficients are just inner products, no need to solve systems of equations.</p></li>
</ul>
<p>This is why orthonormal bases are the most convenient: they make representation and projection effortless.</p>
</section>
<section id="examples-3" class="level4">
<h4 class="anchored" data-anchor-id="examples-3">Examples</h4>
<ol type="1">
<li><p>Standard Basis in <span class="math inline">\(\mathbb{R}^n\)</span>: <span class="math inline">\(\{e_1, e_2, \dots, e_n\}\)</span>, where <span class="math inline">\(e_i\)</span> has 1 in the <span class="math inline">\(i\)</span>-th coordinate and 0 elsewhere.</p>
<ul>
<li>Orthonormal under the standard dot product.</li>
</ul></li>
<li><p>Fourier Basis: Functions <span class="math inline">\(\{\sin(nx), \cos(nx)\}\)</span> on <span class="math inline">\([0,2\pi]\)</span> are orthogonal under the inner product <span class="math inline">\(\langle f,g\rangle = \int_0^{2\pi} f(x)g(x)dx\)</span>.</p>
<ul>
<li>This basis decomposes signals into pure frequencies.</li>
</ul></li>
<li><p>Polynomial Basis: Legendre polynomials <span class="math inline">\(P_n(x)\)</span> are orthogonal on <span class="math inline">\([-1,1]\)</span> with respect to <span class="math inline">\(\langle f,g\rangle = \int_{-1}^1 f(x)g(x)\,dx\)</span>.</p></li>
</ol>
</section>
<section id="geometric-meaning-18" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-18">Geometric Meaning</h4>
<p>Orthogonality splits space into independent “directions.”</p>
<ul>
<li>Orthonormal bases are like perfectly aligned coordinate axes.</li>
<li>Any vector decomposes uniquely as a sum of independent contributions along these axes.</li>
<li>Distances and angles are preserved, making the geometry transparent.</li>
</ul>
</section>
<section id="applications-34" class="level4">
<h4 class="anchored" data-anchor-id="applications-34">Applications</h4>
<ol type="1">
<li>Signal Processing: Decompose signals into orthogonal frequency components.</li>
<li>Machine Learning: Principal components form an orthonormal basis capturing variance directions.</li>
<li>Numerical Methods: Orthonormal bases improve numerical stability.</li>
<li>Quantum Mechanics: States are orthogonal if they represent mutually exclusive outcomes.</li>
<li>Computer Graphics: Rotations are represented by orthogonal matrices with orthonormal columns.</li>
</ol>
</section>
<section id="why-it-matters-68" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-68">Why It Matters</h4>
<ul>
<li>Orthogonality provides independence; orthonormality provides normalization.</li>
<li>Together they make computations, decompositions, and projections clean and efficient.</li>
<li>They underlie Fourier analysis, principal component analysis, and countless modern algorithms.</li>
</ul>
</section>
<section id="try-it-yourself-71" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-71">Try It Yourself</h4>
<ol type="1">
<li>Show that <span class="math inline">\(\{(1,0,0), (0,1,0), (0,0,1)\}\)</span> is an orthonormal basis of <span class="math inline">\(\mathbb{R}^3\)</span>.</li>
<li>Check whether <span class="math inline">\(\{(1,1,0), (1,-1,0), (0,0,1)\}\)</span> is orthonormal under the dot product. If not, normalize it.</li>
<li>Compute the coefficients of <span class="math inline">\(x=(3,4)\)</span> in the basis <span class="math inline">\(\{(1,0), (0,1)\}\)</span> and in the rotated orthonormal basis <span class="math inline">\(\{(1/\sqrt{2}, 1/\sqrt{2}), (-1/\sqrt{2}, 1/\sqrt{2})\}\)</span>.</li>
<li>Challenge: Prove that in any finite-dimensional inner product space, an orthonormal basis always exists (hint: Gram–Schmidt).</li>
</ol>
<p>Orthogonality and orthonormal bases are the backbone of linear algebra: they transform messy problems into elegant decompositions, giving us the cleanest possible language for describing vectors, signals, and data.</p>
</section>
</section>
<section id="gramschmidt-process" class="level3">
<h3 class="anchored" data-anchor-id="gramschmidt-process">73. Gram–Schmidt Process</h3>
<p>The Gram–Schmidt process is a systematic method for turning any linearly independent set of vectors into an orthonormal basis. This process is one of the most elegant bridges between algebra and geometry: it takes arbitrary vectors and makes them mutually perpendicular, while preserving the span.</p>
<section id="the-problem-it-solves" class="level4">
<h4 class="anchored" data-anchor-id="the-problem-it-solves">The Problem It Solves</h4>
<p>Given a set of linearly independent vectors <span class="math inline">\(\{v_1, v_2, \dots, v_n\}\)</span> in an inner product space:</p>
<ul>
<li>They span some subspace <span class="math inline">\(W\)</span>.</li>
<li>But they are not necessarily orthogonal or normalized.</li>
</ul>
<p>Goal: Construct an orthonormal basis <span class="math inline">\(\{u_1, u_2, \dots, u_n\}\)</span> for <span class="math inline">\(W\)</span>.</p>
</section>
<section id="the-gramschmidt-algorithm" class="level4">
<h4 class="anchored" data-anchor-id="the-gramschmidt-algorithm">The Gram–Schmidt Algorithm</h4>
<ol type="1">
<li><p>Start with the first vector:</p>
<p><span class="math display">\[
u_1 = \frac{v_1}{\|v_1\|}.
\]</span></p></li>
<li><p>For the second vector, subtract the projection onto <span class="math inline">\(u_1\)</span>:</p>
<p><span class="math display">\[
w_2 = v_2 - \langle v_2, u_1 \rangle u_1, \quad u_2 = \frac{w_2}{\|w_2\|}.
\]</span></p></li>
<li><p>For the third vector, subtract projections onto both <span class="math inline">\(u_1\)</span> and <span class="math inline">\(u_2\)</span>:</p>
<p><span class="math display">\[
w_3 = v_3 - \langle v_3, u_1 \rangle u_1 - \langle v_3, u_2 \rangle u_2, \quad u_3 = \frac{w_3}{\|w_3\|}.
\]</span></p></li>
<li><p>Continue inductively:</p>
<p><span class="math display">\[
w_k = v_k - \sum_{j=1}^{k-1} \langle v_k, u_j \rangle u_j, \quad u_k = \frac{w_k}{\|w_k\|}.
\]</span></p></li>
</ol>
<p>At each step, <span class="math inline">\(w_k\)</span> is made orthogonal to all previous <span class="math inline">\(u_j\)</span>, and then normalized to form <span class="math inline">\(u_k\)</span>.</p>
</section>
<section id="example-in-mathbbr2-3" class="level4">
<h4 class="anchored" data-anchor-id="example-in-mathbbr2-3">Example in <span class="math inline">\(\mathbb{R}^2\)</span></h4>
<p>Start with <span class="math inline">\(v_1 = (1,1)\)</span>, <span class="math inline">\(v_2 = (1,0)\)</span>.</p>
<ol type="1">
<li><p>Normalize first vector:</p>
<p><span class="math display">\[
u_1 = \frac{(1,1)}{\sqrt{2}} = \left(\tfrac{1}{\sqrt{2}}, \tfrac{1}{\sqrt{2}}\right).
\]</span></p></li>
<li><p>Subtract projection of <span class="math inline">\(v_2\)</span> on <span class="math inline">\(u_1\)</span>:</p>
<p><span class="math display">\[
w_2 = (1,0) - \left(\tfrac{1}{\sqrt{2}}\cdot1 + \tfrac{1}{\sqrt{2}}\cdot0\right)\left(\tfrac{1}{\sqrt{2}}, \tfrac{1}{\sqrt{2}}\right).
\]</span></p>
<p><span class="math display">\[
= (1,0) - \tfrac{1}{\sqrt{2}}\left(\tfrac{1}{\sqrt{2}}, \tfrac{1}{\sqrt{2}}\right).
\]</span></p>
<p><span class="math display">\[
= (1,0) - (0.5,0.5) = (0.5,-0.5).
\]</span></p></li>
<li><p>Normalize:</p>
<p><span class="math display">\[
u_2 = \frac{(0.5,-0.5)}{\sqrt{0.5^2+(-0.5)^2}} = \frac{(0.5,-0.5)}{\sqrt{0.5}} = \left(\tfrac{1}{\sqrt{2}}, -\tfrac{1}{\sqrt{2}}\right).
\]</span></p></li>
</ol>
<p>Final orthonormal basis:</p>
<p><span class="math display">\[
u_1 = \left(\tfrac{1}{\sqrt{2}}, \tfrac{1}{\sqrt{2}}\right), \quad u_2 = \left(\tfrac{1}{\sqrt{2}}, -\tfrac{1}{\sqrt{2}}\right).
\]</span></p>
</section>
<section id="geometric-intuition-5" class="level4">
<h4 class="anchored" data-anchor-id="geometric-intuition-5">Geometric Intuition</h4>
<ul>
<li>Each step removes “overlap” with previously chosen directions.</li>
<li>Think of it as building new perpendicular coordinate axes inside the span of the original vectors.</li>
<li>The result is like rotating and scaling the original set into a perfectly orthogonal system.</li>
</ul>
</section>
<section id="numerical-stability" class="level4">
<h4 class="anchored" data-anchor-id="numerical-stability">Numerical Stability</h4>
<ul>
<li>Classical Gram–Schmidt can suffer from round-off errors in computer calculations.</li>
<li>A numerically stable alternative is Modified Gram–Schmidt (MGS), which reorders the projection steps to reduce loss of orthogonality.</li>
<li>In practice, QR factorization algorithms often implement MGS or Householder reflections.</li>
</ul>
</section>
<section id="applications-35" class="level4">
<h4 class="anchored" data-anchor-id="applications-35">Applications</h4>
<ol type="1">
<li>QR Factorization: Gram–Schmidt provides the foundation: <span class="math inline">\(A = QR\)</span>, where <span class="math inline">\(Q\)</span> is orthogonal and <span class="math inline">\(R\)</span> is upper triangular.</li>
<li>Data Compression: Orthonormal bases from Gram–Schmidt lead to efficient representations.</li>
<li>Signal Processing: Ensures independent frequency or wave components.</li>
<li>Machine Learning: Used in orthogonalization of features and dimensionality reduction.</li>
<li>Physics: Orthogonal states in quantum mechanics can be constructed from arbitrary states using Gram–Schmidt.</li>
</ol>
</section>
<section id="why-it-matters-69" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-69">Why It Matters</h4>
<ul>
<li>Gram–Schmidt guarantees that any independent set can be reshaped into an orthonormal basis.</li>
<li>It underlies computational methods like QR decomposition, least squares, and numerical PDE solvers.</li>
<li>It makes projections, coordinates, and orthogonality explicit and manageable.</li>
</ul>
</section>
<section id="try-it-yourself-72" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-72">Try It Yourself</h4>
<ol type="1">
<li>Apply Gram–Schmidt to <span class="math inline">\((1,0,1)\)</span>, <span class="math inline">\((1,1,0)\)</span>, <span class="math inline">\((0,1,1)\)</span> in <span class="math inline">\(\mathbb{R}^3\)</span>. Verify orthonormality.</li>
<li>Show that the span of the orthonormal basis equals the span of the original vectors.</li>
<li>Use Gram–Schmidt to find an orthonormal basis for polynomials <span class="math inline">\(\{1,x,x^2\}\)</span> on <span class="math inline">\([-1,1]\)</span> with inner product <span class="math inline">\(\langle f,g\rangle = \int_{-1}^1 f(x)g(x)\,dx\)</span>.</li>
<li>Challenge: Prove that Gram–Schmidt always works for linearly independent sets, but fails if the set is dependent.</li>
</ol>
<p>The Gram–Schmidt process is the algorithmic heart of orthogonality: it takes the messy and redundant and reshapes it into clean, perpendicular building blocks for the spaces we study.</p>
</section>
</section>
<section id="projections-onto-subspaces" class="level3">
<h3 class="anchored" data-anchor-id="projections-onto-subspaces">74. Projections onto Subspaces</h3>
<p>Projections are a natural extension of orthogonality: they describe how to “drop” a vector onto a subspace in the most natural way, minimizing the distance. Understanding projections is crucial for solving least squares problems, decomposing vectors, and interpreting data in terms of simpler, lower-dimensional structures.</p>
<section id="projection-onto-a-vector" class="level4">
<h4 class="anchored" data-anchor-id="projection-onto-a-vector">Projection onto a Vector</h4>
<p>Start with the simplest case: projecting a vector <span class="math inline">\(x\)</span> onto a nonzero vector <span class="math inline">\(u\)</span>.</p>
<ol type="1">
<li><p>The projection is the component of <span class="math inline">\(x\)</span> that lies in the direction of <span class="math inline">\(u\)</span>.</p></li>
<li><p>Formula:</p>
<p><span class="math display">\[
\text{proj}_u(x) = \frac{\langle x, u \rangle}{\langle u, u \rangle} u.
\]</span></p>
<p>If <span class="math inline">\(u\)</span> is normalized (<span class="math inline">\(\|u\|=1\)</span>), this simplifies to</p>
<p><span class="math display">\[
\text{proj}_u(x) = \langle x, u \rangle u.
\]</span></p></li>
</ol>
<p>Geometrically, this is the foot of the perpendicular from <span class="math inline">\(x\)</span> onto the line spanned by <span class="math inline">\(u\)</span>.</p>
</section>
<section id="projection-onto-an-orthonormal-basis" class="level4">
<h4 class="anchored" data-anchor-id="projection-onto-an-orthonormal-basis">Projection onto an Orthonormal Basis</h4>
<p>Suppose we have an orthonormal basis <span class="math inline">\(\{u_1, u_2, \dots, u_k\}\)</span> for a subspace <span class="math inline">\(W\)</span>. Then the projection of <span class="math inline">\(x\)</span> onto <span class="math inline">\(W\)</span> is:</p>
<p><span class="math display">\[
\text{proj}_W(x) = \sum_{i=1}^k \langle x, u_i \rangle u_i.
\]</span></p>
<p>This formula is powerful:</p>
<ul>
<li>Each coefficient <span class="math inline">\(\langle x, u_i \rangle\)</span> captures how much of <span class="math inline">\(x\)</span> aligns with <span class="math inline">\(u_i\)</span>.</li>
<li>The sum reconstructs the best approximation of <span class="math inline">\(x\)</span> inside <span class="math inline">\(W\)</span>.</li>
</ul>
</section>
<section id="projection-matrix" class="level4">
<h4 class="anchored" data-anchor-id="projection-matrix">Projection Matrix</h4>
<p>When working in coordinates, projections can be represented by matrices.</p>
<ul>
<li><p>If <span class="math inline">\(U\)</span> is the <span class="math inline">\(n \times k\)</span> matrix with orthonormal columns <span class="math inline">\(\{u_1, \dots, u_k\}\)</span>, then</p>
<p><span class="math display">\[
P = UU^T
\]</span></p>
<p>is the projection matrix onto <span class="math inline">\(W\)</span>.</p></li>
</ul>
<p>Properties of <span class="math inline">\(P\)</span>:</p>
<ol type="1">
<li>Idempotence: <span class="math inline">\(P^2 = P\)</span>.</li>
<li>Symmetry: <span class="math inline">\(P^T = P\)</span>.</li>
<li>Best approximation: For any <span class="math inline">\(x\)</span>, <span class="math inline">\(\|x - Px\|\)</span> is minimized.</li>
</ol>
</section>
<section id="projection-and-orthogonal-complements" class="level4">
<h4 class="anchored" data-anchor-id="projection-and-orthogonal-complements">Projection and Orthogonal Complements</h4>
<p>If <span class="math inline">\(W\)</span> is a subspace of <span class="math inline">\(V\)</span>, then every vector <span class="math inline">\(x \in V\)</span> can be decomposed uniquely as</p>
<p><span class="math display">\[
x = \text{proj}_W(x) + \text{proj}_{W^\perp}(x),
\]</span></p>
<p>where <span class="math inline">\(W^\perp\)</span> is the orthogonal complement of <span class="math inline">\(W\)</span>.</p>
<p>This decomposition is the orthogonal decomposition theorem. It says: space splits cleanly into “in” and “out of” components relative to a subspace.</p>
</section>
<section id="example-in-mathbbr2-4" class="level4">
<h4 class="anchored" data-anchor-id="example-in-mathbbr2-4">Example in <span class="math inline">\(\mathbb{R}^2\)</span></h4>
<p>Let <span class="math inline">\(u = (2,1)\)</span>, and project <span class="math inline">\(x = (3,4)\)</span> onto span<span class="math inline">\(\{u\}\)</span>.</p>
<ol type="1">
<li><p>Compute inner product: <span class="math inline">\(\langle x,u\rangle = 3\cdot 2 + 4\cdot 1 = 10\)</span>.</p></li>
<li><p>Compute norm squared: <span class="math inline">\(\langle u,u\rangle = 2^2 + 1^2 = 5\)</span>.</p></li>
<li><p>Projection:</p>
<p><span class="math display">\[
\text{proj}_u(x) = \frac{10}{5}(2,1) = 2(2,1) = (4,2).
\]</span></p></li>
<li><p>Orthogonal error:</p>
<p><span class="math display">\[
x - \text{proj}_u(x) = (3,4) - (4,2) = (-1,2).
\]</span></p></li>
</ol>
<p>Notice: <span class="math inline">\((4,2)\)</span> lies on the line through <span class="math inline">\(u\)</span>, and the error vector <span class="math inline">\((-1,2)\)</span> is orthogonal to <span class="math inline">\(u\)</span>.</p>
</section>
<section id="applications-36" class="level4">
<h4 class="anchored" data-anchor-id="applications-36">Applications</h4>
<ol type="1">
<li>Least Squares Regression: The regression line is the projection of data onto the subspace spanned by predictor variables.</li>
<li>Dimensionality Reduction: Principal Component Analysis (PCA) projects data onto the subspace of top eigenvectors.</li>
<li>Computer Graphics: 3D objects are projected onto 2D screens.</li>
<li>Numerical Methods: Projections solve equations approximately when exact solutions don’t exist.</li>
<li>Physics: Work and energy are computed via projections of forces and velocities.</li>
</ol>
</section>
<section id="why-it-matters-70" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-70">Why It Matters</h4>
<ul>
<li>Projections are the essence of approximation: they give the “best possible” version of a vector inside a chosen subspace.</li>
<li>They formalize independence: the error vector is always orthogonal to the subspace.</li>
<li>They provide geometric intuition for statistics, machine learning, and numerical computation.</li>
</ul>
</section>
<section id="try-it-yourself-73" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-73">Try It Yourself</h4>
<ol type="1">
<li>Compute the projection of <span class="math inline">\(x = (2,3,4)\)</span> onto <span class="math inline">\(u = (1,1,1)\)</span>.</li>
<li>Verify that the residual <span class="math inline">\(x - \text{proj}_u(x)\)</span> is orthogonal to <span class="math inline">\(u\)</span>.</li>
<li>Write the projection matrix for the subspace spanned by <span class="math inline">\(\{(1,0,0),(0,1,0)\}\)</span> in <span class="math inline">\(\mathbb{R}^3\)</span>.</li>
<li>Challenge: Prove that projection matrices are idempotent and symmetric.</li>
</ol>
<p>Projections turn vector spaces into cleanly split components: what lies “inside” a subspace and what lies “outside.” This idea, simple yet profound, runs through geometry, data analysis, and physics alike.</p>
</section>
</section>
<section id="orthogonal-decomposition-theorem" class="level3">
<h3 class="anchored" data-anchor-id="orthogonal-decomposition-theorem">75. Orthogonal Decomposition Theorem</h3>
<p>One of the cornerstones of linear algebra is the orthogonal decomposition theorem, which states that every vector in an inner product space can be uniquely split into two parts: one lying inside a subspace and the other lying in its orthogonal complement. This gives us a clear way to organize information, separate influences, and simplify computations.</p>
<section id="statement-of-the-theorem-1" class="level4">
<h4 class="anchored" data-anchor-id="statement-of-the-theorem-1">Statement of the Theorem</h4>
<p>Let <span class="math inline">\(V\)</span> be an inner product space and <span class="math inline">\(W\)</span> a subspace of <span class="math inline">\(V\)</span>. Then for every vector <span class="math inline">\(x \in V\)</span>, there exist unique vectors <span class="math inline">\(w \in W\)</span> and <span class="math inline">\(z \in W^\perp\)</span> such that</p>
<p><span class="math display">\[
x = w + z.
\]</span></p>
<p>Here:</p>
<ul>
<li><span class="math inline">\(w = \text{proj}_W(x)\)</span>, the projection of <span class="math inline">\(x\)</span> onto <span class="math inline">\(W\)</span>.</li>
<li><span class="math inline">\(z = x - \text{proj}_W(x)\)</span>, the orthogonal component.</li>
</ul>
<p>This decomposition is unique: no other pair of vectors from <span class="math inline">\(W\)</span> and <span class="math inline">\(W^\perp\)</span> adds up to <span class="math inline">\(x\)</span>.</p>
</section>
<section id="example-in-mathbbr2-5" class="level4">
<h4 class="anchored" data-anchor-id="example-in-mathbbr2-5">Example in <span class="math inline">\(\mathbb{R}^2\)</span></h4>
<p>Take <span class="math inline">\(W\)</span> to be the line spanned by <span class="math inline">\(u = (1,2)\)</span>. For <span class="math inline">\(x = (4,1)\)</span>:</p>
<ol type="1">
<li><p>Projection:</p>
<p><span class="math display">\[
\text{proj}_u(x) = \frac{\langle x,u \rangle}{\langle u,u \rangle} u.
\]</span></p>
<p>Compute: <span class="math inline">\(\langle x,u\rangle = 4\cdot 1 + 1\cdot 2 = 6\)</span>, and <span class="math inline">\(\langle u,u\rangle = 1^2+2^2=5\)</span>. So</p>
<p><span class="math display">\[
\text{proj}_u(x) = \frac{6}{5}(1,2) = \left(\tfrac{6}{5}, \tfrac{12}{5}\right).
\]</span></p></li>
<li><p>Orthogonal component:</p>
<p><span class="math display">\[
z = x - \text{proj}_u(x) = (4,1) - \left(\tfrac{6}{5}, \tfrac{12}{5}\right) = \left(\tfrac{14}{5}, -\tfrac{7}{5}\right).
\]</span></p></li>
<li><p>Verify: <span class="math inline">\(\langle u, z\rangle = 1\cdot \tfrac{14}{5} + 2\cdot (-\tfrac{7}{5}) = 0\)</span>. Thus, <span class="math inline">\(z \in W^\perp\)</span>.</p></li>
</ol>
<p>So we have</p>
<p><span class="math display">\[
x = \underbrace{\left(\tfrac{6}{5}, \tfrac{12}{5}\right)}_{\in W} + \underbrace{\left(\tfrac{14}{5}, -\tfrac{7}{5}\right)}_{\in W^\perp}.
\]</span></p>
</section>
<section id="geometric-meaning-19" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-19">Geometric Meaning</h4>
<ul>
<li>The decomposition splits <span class="math inline">\(x\)</span> into its “in-subspace” part and its “out-of-subspace” part.</li>
<li><span class="math inline">\(w\)</span> is the closest point in <span class="math inline">\(W\)</span> to <span class="math inline">\(x\)</span>.</li>
<li><span class="math inline">\(z\)</span> is the leftover “error,” always perpendicular to <span class="math inline">\(W\)</span>.</li>
</ul>
<p>Geometrically, the shortest path from <span class="math inline">\(x\)</span> to a subspace is always orthogonal.</p>
</section>
<section id="orthogonal-complements" class="level4">
<h4 class="anchored" data-anchor-id="orthogonal-complements">Orthogonal Complements</h4>
<ul>
<li><p>The orthogonal complement <span class="math inline">\(W^\perp\)</span> contains all vectors orthogonal to every vector in <span class="math inline">\(W\)</span>.</p></li>
<li><p>Dimensional relationship:</p>
<p><span class="math display">\[
\dim(W) + \dim(W^\perp) = \dim(V).
\]</span></p></li>
<li><p>Together, <span class="math inline">\(W\)</span> and <span class="math inline">\(W^\perp\)</span> partition the space <span class="math inline">\(V\)</span>.</p></li>
</ul>
</section>
<section id="projection-matrices-and-decomposition" class="level4">
<h4 class="anchored" data-anchor-id="projection-matrices-and-decomposition">Projection Matrices and Decomposition</h4>
<p>If <span class="math inline">\(P\)</span> is the projection matrix onto <span class="math inline">\(W\)</span>:</p>
<p><span class="math display">\[
x = Px + (I-P)x,
\]</span></p>
<p>where <span class="math inline">\(Px \in W\)</span> and <span class="math inline">\((I-P)x \in W^\perp\)</span>.</p>
<p>This formulation is used constantly in numerical linear algebra.</p>
</section>
<section id="applications-37" class="level4">
<h4 class="anchored" data-anchor-id="applications-37">Applications</h4>
<ol type="1">
<li>Least Squares Approximation: The best-fit solution is the projection; the error lies in the orthogonal complement.</li>
<li>Fourier Analysis: Any signal decomposes into a sum of components along orthogonal basis functions plus residuals.</li>
<li>Statistics: Regression decomposes data into explained variance (in the subspace of predictors) and residual variance (orthogonal).</li>
<li>Engineering: Splitting forces into parallel and perpendicular components relative to a surface.</li>
<li>Computer Graphics: Decomposing movement into screen-plane projection and depth (orthogonal direction).</li>
</ol>
</section>
<section id="why-it-matters-71" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-71">Why It Matters</h4>
<ul>
<li>Orthogonal decomposition gives clarity: every vector splits into “relevant” and “irrelevant” parts relative to a chosen subspace.</li>
<li>It provides the foundation for least squares, regression, and signal approximation.</li>
<li>It ensures uniqueness, stability, and interpretability in vector computations.</li>
</ul>
</section>
<section id="try-it-yourself-74" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-74">Try It Yourself</h4>
<ol type="1">
<li>In <span class="math inline">\(\mathbb{R}^3\)</span>, decompose <span class="math inline">\(x = (1,2,3)\)</span> into components in span<span class="math inline">\((1,0,0)\)</span> and its orthogonal complement.</li>
<li>Show that if <span class="math inline">\(W\)</span> is spanned by <span class="math inline">\((1,1,0)\)</span> and <span class="math inline">\((0,1,1)\)</span>, then any vector in <span class="math inline">\(\mathbb{R}^3\)</span> can be uniquely split into <span class="math inline">\(W\)</span> and <span class="math inline">\(W^\perp\)</span>.</li>
<li>Write down the projection matrix <span class="math inline">\(P\)</span> for <span class="math inline">\(W = \text{span}\{(1,0,0),(0,1,0)\}\)</span> in <span class="math inline">\(\mathbb{R}^3\)</span>. Verify that <span class="math inline">\(I-P\)</span> projects onto <span class="math inline">\(W^\perp\)</span>.</li>
<li>Challenge: Prove the orthogonal decomposition theorem using projection matrices and the fact that <span class="math inline">\(P^2 = P\)</span>.</li>
</ol>
<p>The orthogonal decomposition theorem guarantees that every vector finds its closest approximation in a chosen subspace and a perfectly perpendicular remainder-an elegant structure that makes analysis and computation possible in countless domains.</p>
</section>
</section>
<section id="orthogonal-projections-and-least-squares" class="level3">
<h3 class="anchored" data-anchor-id="orthogonal-projections-and-least-squares">76. Orthogonal Projections and Least Squares</h3>
<p>One of the deepest connections in linear algebra is between orthogonal projections and the least squares method. When equations don’t have an exact solution, least squares finds the “best approximate” one. The theory behind it is entirely geometric: the best solution is the projection of a vector onto a subspace.</p>
<section id="the-setup-overdetermined-systems" class="level4">
<h4 class="anchored" data-anchor-id="the-setup-overdetermined-systems">The Setup: Overdetermined Systems</h4>
<p>Consider a system of equations <span class="math inline">\(Ax = b\)</span>, where</p>
<ul>
<li><span class="math inline">\(A\)</span> is an <span class="math inline">\(m \times n\)</span> matrix with <span class="math inline">\(m &gt; n\)</span> (more equations than unknowns).</li>
<li><span class="math inline">\(b \in \mathbb{R}^m\)</span> may not lie in the column space of <span class="math inline">\(A\)</span>.</li>
</ul>
<p>This means:</p>
<ul>
<li>There may be no exact solution.</li>
<li>Instead, we want <span class="math inline">\(x\)</span> that makes <span class="math inline">\(Ax\)</span> as close as possible to <span class="math inline">\(b\)</span>.</li>
</ul>
</section>
<section id="least-squares-problem" class="level4">
<h4 class="anchored" data-anchor-id="least-squares-problem">Least Squares Problem</h4>
<p>The least squares solution minimizes the error:</p>
<p><span class="math display">\[
\min_x \|Ax - b\|^2.
\]</span></p>
<p>Here:</p>
<ul>
<li><span class="math inline">\(Ax\)</span> is the projection of <span class="math inline">\(b\)</span> onto the column space of <span class="math inline">\(A\)</span>.</li>
<li>The error vector <span class="math inline">\(b - Ax\)</span> is orthogonal to the column space.</li>
</ul>
<p>This is exactly the orthogonal decomposition theorem applied to <span class="math inline">\(b\)</span>.</p>
</section>
<section id="derivation-of-normal-equations" class="level4">
<h4 class="anchored" data-anchor-id="derivation-of-normal-equations">Derivation of Normal Equations</h4>
<p>We want <span class="math inline">\(r = b - Ax\)</span> to be orthogonal to every column of <span class="math inline">\(A\)</span>:</p>
<p><span class="math display">\[
A^T (b - Ax) = 0.
\]</span></p>
<p>Rearranging:</p>
<p><span class="math display">\[
A^T A x = A^T b.
\]</span></p>
<p>This system is called the normal equations. Its solution <span class="math inline">\(x\)</span> gives the least squares approximation.</p>
</section>
<section id="projection-matrix-in-least-squares" class="level4">
<h4 class="anchored" data-anchor-id="projection-matrix-in-least-squares">Projection Matrix in Least Squares</h4>
<p>The projection of <span class="math inline">\(b\)</span> onto <span class="math inline">\(\text{Col}(A)\)</span> is</p>
<p><span class="math display">\[
\hat{b} = A(A^T A)^{-1} A^T b,
\]</span></p>
<p>assuming <span class="math inline">\(A^T A\)</span> is invertible.</p>
<p>Here,</p>
<ul>
<li><span class="math inline">\(P = A(A^T A)^{-1} A^T\)</span> is the projection matrix onto the column space of <span class="math inline">\(A\)</span>.</li>
<li>The fitted vector is <span class="math inline">\(\hat{b} = Pb\)</span>.</li>
<li>The residual <span class="math inline">\(r = b - \hat{b}\)</span> lies in the orthogonal complement of <span class="math inline">\(\text{Col}(A)\)</span>.</li>
</ul>
</section>
<section id="example-2" class="level4">
<h4 class="anchored" data-anchor-id="example-2">Example</h4>
<p>Suppose <span class="math inline">\(A = \begin{bmatrix}1 \\ 2 \\ 3\end{bmatrix}\)</span>, <span class="math inline">\(b = \begin{bmatrix}2 \\ 2 \\ 4\end{bmatrix}\)</span>.</p>
<ul>
<li><p>Column space of <span class="math inline">\(A\)</span>: span of <span class="math inline">\((1,2,3)\)</span>.</p></li>
<li><p>Projection formula:</p>
<p><span class="math display">\[
\hat{b} = \frac{\langle b, A \rangle}{\langle A, A \rangle} A.
\]</span></p></li>
<li><p>Compute: <span class="math inline">\(\langle b,A\rangle = 2\cdot1+2\cdot2+4\cdot3 = 18\)</span>. <span class="math inline">\(\langle A,A\rangle = 1^2+2^2+3^2=14\)</span>.</p></li>
<li><p>Projection:</p>
<p><span class="math display">\[
\hat{b} = \frac{18}{14}(1,2,3) = \left(\tfrac{9}{7}, \tfrac{18}{7}, \tfrac{27}{7}\right).
\]</span></p></li>
<li><p>Residual:</p>
<p><span class="math display">\[
r = b - \hat{b} = \left(\tfrac{5}{7}, -\tfrac{4}{7}, \tfrac{1}{7}\right).
\]</span></p></li>
</ul>
<p>Check: <span class="math inline">\(\langle r,A\rangle = 0\)</span>, so it’s orthogonal.</p>
</section>
<section id="geometric-meaning-20" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-20">Geometric Meaning</h4>
<ul>
<li>The least squares solution is the point in <span class="math inline">\(\text{Col}(A)\)</span> closest to <span class="math inline">\(b\)</span>.</li>
<li>The error vector is orthogonal to the subspace.</li>
<li>This is like dropping a perpendicular from <span class="math inline">\(b\)</span> to the subspace <span class="math inline">\(\text{Col}(A)\)</span>.</li>
</ul>
</section>
<section id="applications-38" class="level4">
<h4 class="anchored" data-anchor-id="applications-38">Applications</h4>
<ol type="1">
<li>Statistics: Linear regression uses least squares to fit models to data.</li>
<li>Engineering: Curve fitting, system identification, and calibration.</li>
<li>Computer Graphics: Best-fit transformations (e.g., Procrustes analysis).</li>
<li>Machine Learning: Optimization of linear models (before moving to nonlinear methods).</li>
<li>Numerical Methods: Solving inconsistent systems of equations.</li>
</ol>
</section>
<section id="why-it-matters-72" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-72">Why It Matters</h4>
<ul>
<li>Orthogonal projections explain why least squares gives the best approximation.</li>
<li>They reveal the geometry behind regression: data is projected onto the model space.</li>
<li>They connect linear algebra with statistics, optimization, and applied sciences.</li>
</ul>
</section>
<section id="try-it-yourself-75" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-75">Try It Yourself</h4>
<ol type="1">
<li>Solve <span class="math inline">\(\min_x \|Ax-b\|\)</span> for <span class="math inline">\(A = \begin{bmatrix}1 &amp; 1 \\ 1 &amp; 2 \\ 1 &amp; 3\end{bmatrix}\)</span>, <span class="math inline">\(b=(1,2,2)^T\)</span>. Interpret the result.</li>
<li>Derive the projection matrix <span class="math inline">\(P\)</span> for this system.</li>
<li>Show that the residual is orthogonal to each column of <span class="math inline">\(A\)</span>.</li>
<li>Challenge: Prove that among all possible approximations <span class="math inline">\(Ax\)</span>, the least squares solution is unique if and only if <span class="math inline">\(A^T A\)</span> is invertible.</li>
</ol>
<p>Orthogonal projections turn the messy, unsolvable world of overdetermined equations into one of best possible approximations. Least squares is not just an algebraic trick-it is the geometric essence of “closeness” in higher-dimensional spaces.</p>
</section>
</section>
<section id="qr-decomposition" class="level3">
<h3 class="anchored" data-anchor-id="qr-decomposition">77. QR Decomposition</h3>
<p>QR decomposition is a factorization of a matrix into an orthogonal part and a triangular part. It grows directly out of orthogonality and the Gram–Schmidt process, and it plays a central role in numerical linear algebra, providing a stable and efficient way to solve systems, compute least squares solutions, and analyze matrices.</p>
<section id="definition-4" class="level4">
<h4 class="anchored" data-anchor-id="definition-4">Definition</h4>
<p>For a real <span class="math inline">\(m \times n\)</span> matrix <span class="math inline">\(A\)</span> with linearly independent columns:</p>
<p><span class="math display">\[
A = QR,
\]</span></p>
<p>where:</p>
<ul>
<li><span class="math inline">\(Q\)</span> is an <span class="math inline">\(m \times n\)</span> matrix with orthonormal columns (<span class="math inline">\(Q^T Q = I\)</span>).</li>
<li><span class="math inline">\(R\)</span> is an <span class="math inline">\(n \times n\)</span> upper triangular matrix.</li>
</ul>
<p>This decomposition is unique if we require <span class="math inline">\(R\)</span> to have positive diagonal entries.</p>
</section>
<section id="connection-to-gramschmidt" class="level4">
<h4 class="anchored" data-anchor-id="connection-to-gramschmidt">Connection to Gram–Schmidt</h4>
<p>The Gram–Schmidt process applied to the columns of <span class="math inline">\(A\)</span> produces the orthonormal columns of <span class="math inline">\(Q\)</span>. The coefficients used during the orthogonalization steps naturally form the entries of <span class="math inline">\(R\)</span>.</p>
<ul>
<li>Each column of <span class="math inline">\(A\)</span> is expressed as a combination of the orthonormal columns of <span class="math inline">\(Q\)</span>.</li>
<li>The coefficients of this expression populate the triangular matrix <span class="math inline">\(R\)</span>.</li>
</ul>
</section>
<section id="example-3" class="level4">
<h4 class="anchored" data-anchor-id="example-3">Example</h4>
<p>Let</p>
<p><span class="math display">\[
A = \begin{bmatrix} 1 &amp; 1 \\ 1 &amp; 0 \\ 0 &amp; 1 \end{bmatrix}.
\]</span></p>
<ol type="1">
<li><p>Apply Gram–Schmidt to the columns:</p>
<ul>
<li><p><span class="math inline">\(v_1 = (1,1,0)^T\)</span>, normalize:</p>
<p><span class="math display">\[
u_1 = \frac{1}{\sqrt{2}}(1,1,0)^T.
\]</span></p></li>
<li><p>Subtract projection from <span class="math inline">\(v_2=(1,0,1)^T\)</span>:</p>
<p><span class="math display">\[
w_2 = v_2 - \langle v_2,u_1\rangle u_1.
\]</span></p>
<p>Compute <span class="math inline">\(\langle v_2,u_1\rangle = \tfrac{1}{\sqrt{2}}(1+0+0)=\tfrac{1}{\sqrt{2}}\)</span>. So</p>
<p><span class="math display">\[
w_2 = (1,0,1)^T - \tfrac{1}{\sqrt{2}}(1,1,0)^T = \left(\tfrac{1}{2}, -\tfrac{1}{2}, 1\right)^T.
\]</span></p>
<p>Normalize:</p>
<p><span class="math display">\[
u_2 = \frac{1}{\sqrt{1.5}} \left(\tfrac{1}{2}, -\tfrac{1}{2}, 1\right)^T.
\]</span></p></li>
</ul></li>
<li><p>Construct <span class="math inline">\(Q = [u_1, u_2]\)</span>.</p></li>
<li><p>Compute <span class="math inline">\(R = Q^T A\)</span>.</p></li>
</ol>
<p>The result is <span class="math inline">\(A = QR\)</span>, with <span class="math inline">\(Q\)</span> orthonormal and <span class="math inline">\(R\)</span> triangular.</p>
</section>
<section id="geometric-meaning-21" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-21">Geometric Meaning</h4>
<ul>
<li><span class="math inline">\(Q\)</span> represents an orthogonal change of basis-rotations and reflections that preserve length and angle.</li>
<li><span class="math inline">\(R\)</span> encodes scaling and shear in the new orthonormal coordinate system.</li>
<li>Together, they show how <span class="math inline">\(A\)</span> transforms space: first rotate into a clean basis, then apply triangular distortion.</li>
</ul>
</section>
<section id="applications-39" class="level4">
<h4 class="anchored" data-anchor-id="applications-39">Applications</h4>
<ol type="1">
<li><p>Least Squares: Instead of solving <span class="math inline">\(A^T A x = A^T b\)</span>, we use <span class="math inline">\(QR\)</span>:</p>
<p><span class="math display">\[
Ax = b \quad \Rightarrow \quad QRx = b.
\]</span></p>
<p>Multiply by <span class="math inline">\(Q^T\)</span>:</p>
<p><span class="math display">\[
Rx = Q^T b.
\]</span></p>
<p>Since <span class="math inline">\(R\)</span> is triangular, solving for <span class="math inline">\(x\)</span> is efficient and numerically stable.</p></li>
<li><p>Eigenvalue Algorithms: The QR algorithm iteratively applies QR factorizations to approximate eigenvalues.</p></li>
<li><p>Numerical Stability: Orthogonal transformations minimize numerical errors compared to solving normal equations.</p></li>
<li><p>Machine Learning: Many algorithms (e.g., linear regression, PCA) use QR decomposition for efficiency and stability.</p></li>
<li><p>Computer Graphics: Orthogonal factors preserve shapes; triangular factors simplify transformations.</p></li>
</ol>
</section>
<section id="why-it-matters-73" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-73">Why It Matters</h4>
<ul>
<li>QR decomposition bridges theory (Gram–Schmidt orthogonalization) and computation (matrix factorization).</li>
<li>It avoids pitfalls of normal equations, improving numerical stability.</li>
<li>It underpins algorithms across statistics, engineering, and computer science.</li>
</ul>
</section>
<section id="try-it-yourself-76" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-76">Try It Yourself</h4>
<ol type="1">
<li><p>Compute the QR decomposition of</p>
<p><span class="math display">\[
A = \begin{bmatrix}1 &amp; 2 \\ 2 &amp; 3 \\ 4 &amp; 5\end{bmatrix}.
\]</span></p></li>
<li><p>Verify that <span class="math inline">\(Q^T Q = I\)</span> and <span class="math inline">\(R\)</span> is upper triangular.</p></li>
<li><p>Use QR to solve the least squares problem <span class="math inline">\(Ax \approx b\)</span> with <span class="math inline">\(b=(1,1,1)^T\)</span>.</p></li>
<li><p>Challenge: Show that if <span class="math inline">\(A\)</span> is square and orthogonal, then <span class="math inline">\(R=I\)</span> and <span class="math inline">\(Q=A\)</span>.</p></li>
</ol>
<p>QR decomposition turns the messy process of solving least squares into a clean, geometric procedure-rotating into a better coordinate system before solving. It is one of the most powerful tools in the linear algebra toolkit.</p>
</section>
</section>
<section id="orthogonal-matrices" class="level3">
<h3 class="anchored" data-anchor-id="orthogonal-matrices">78. Orthogonal Matrices</h3>
<p>Orthogonal matrices are square matrices whose rows and columns form an orthonormal set. They are the algebraic counterpart of rigid motions in geometry: transformations that preserve lengths, angles, and orientation (except for reflections).</p>
<section id="definition-5" class="level4">
<h4 class="anchored" data-anchor-id="definition-5">Definition</h4>
<p>A square matrix <span class="math inline">\(Q \in \mathbb{R}^{n \times n}\)</span> is orthogonal if</p>
<p><span class="math display">\[
Q^T Q = QQ^T = I.
\]</span></p>
<p>This means:</p>
<ul>
<li>The columns of <span class="math inline">\(Q\)</span> are orthonormal.</li>
<li>The rows of <span class="math inline">\(Q\)</span> are also orthonormal.</li>
</ul>
</section>
<section id="properties" class="level4">
<h4 class="anchored" data-anchor-id="properties">Properties</h4>
<ol type="1">
<li><p>Inverse Equals Transpose:</p>
<p><span class="math display">\[
Q^{-1} = Q^T.
\]</span></p>
<p>This makes orthogonal matrices especially easy to invert.</p></li>
<li><p>Preservation of Norms: For any vector <span class="math inline">\(x\)</span>,</p>
<p><span class="math display">\[
\|Qx\| = \|x\|.
\]</span></p>
<p>Orthogonal transformations never stretch or shrink vectors.</p></li>
<li><p>Preservation of Inner Products:</p>
<p><span class="math display">\[
\langle Qx, Qy \rangle = \langle x, y \rangle.
\]</span></p>
<p>Angles are preserved.</p></li>
<li><p>Determinant: <span class="math inline">\(\det(Q) = \pm 1\)</span>.</p>
<ul>
<li>If <span class="math inline">\(\det(Q) = 1\)</span>, <span class="math inline">\(Q\)</span> is a rotation.</li>
<li>If <span class="math inline">\(\det(Q) = -1\)</span>, <span class="math inline">\(Q\)</span> is a reflection combined with rotation.</li>
</ul></li>
</ol>
</section>
<section id="examples-4" class="level4">
<h4 class="anchored" data-anchor-id="examples-4">Examples</h4>
<ol type="1">
<li><p>2D Rotation Matrix:</p>
<p><span class="math display">\[
Q = \begin{bmatrix} \cos \theta &amp; -\sin \theta \\ \sin \theta &amp; \cos \theta \end{bmatrix}.
\]</span></p>
<p>Rotates vectors by angle <span class="math inline">\(\theta\)</span>.</p></li>
<li><p>2D Reflection Matrix: Reflection across the <span class="math inline">\(x\)</span>-axis:</p>
<p><span class="math display">\[
Q = \begin{bmatrix} 1 &amp; 0 \\ 0 &amp; -1 \end{bmatrix}.
\]</span></p></li>
<li><p>Permutation Matrices: Swapping coordinates is orthogonal because it preserves lengths. Example in 3D:</p>
<p><span class="math display">\[
Q = \begin{bmatrix}0 &amp; 1 &amp; 0 \\ 1 &amp; 0 &amp; 0 \\ 0 &amp; 0 &amp; 1\end{bmatrix}.
\]</span></p></li>
</ol>
</section>
<section id="geometric-meaning-22" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-22">Geometric Meaning</h4>
<p>Orthogonal matrices represent isometries: transformations that preserve the shape of objects.</p>
<ul>
<li>They can rotate, reflect, or permute axes.</li>
<li>They never distort lengths or angles.</li>
</ul>
<p>This is why in computer graphics, orthogonal matrices model pure rotations and reflections without scaling.</p>
</section>
<section id="applications-40" class="level4">
<h4 class="anchored" data-anchor-id="applications-40">Applications</h4>
<ol type="1">
<li>Computer Graphics: Rotations of 3D models use orthogonal matrices to avoid distortion.</li>
<li>Numerical Linear Algebra: Orthogonal transformations are numerically stable, widely used in QR factorization and eigenvalue algorithms.</li>
<li>Data Compression: Orthogonal transforms like the Fourier and cosine transforms preserve energy.</li>
<li>Signal Processing: Orthogonal filters separate signals into independent components.</li>
<li>Physics: Orthogonal matrices describe rotations in rigid body dynamics.</li>
</ol>
</section>
<section id="why-it-matters-74" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-74">Why It Matters</h4>
<ul>
<li>Orthogonal matrices are the building blocks of stable algorithms.</li>
<li>They describe symmetry, structure, and invariance in physical and computational systems.</li>
<li>They serve as the simplest and most powerful class of transformations that preserve geometry exactly.</li>
</ul>
</section>
<section id="try-it-yourself-77" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-77">Try It Yourself</h4>
<ol type="1">
<li><p>Verify that</p>
<p><span class="math display">\[
Q = \begin{bmatrix}0 &amp; -1 \\ 1 &amp; 0\end{bmatrix}
\]</span></p>
<p>is orthogonal. What geometric transformation does it represent?</p></li>
<li><p>Prove that the determinant of an orthogonal matrix must be <span class="math inline">\(\pm 1\)</span>.</p></li>
<li><p>Show that multiplying two orthogonal matrices gives another orthogonal matrix.</p></li>
<li><p>Challenge: Prove that eigenvalues of orthogonal matrices lie on the complex unit circle (i.e., <span class="math inline">\(|\lambda|=1\)</span>).</p></li>
</ol>
<p>Orthogonal matrices capture the essence of symmetry: transformations that preserve structure exactly. They lie at the heart of geometry, physics, and computation.</p>
</section>
</section>
<section id="fourier-viewpoint" class="level3">
<h3 class="anchored" data-anchor-id="fourier-viewpoint">79. Fourier Viewpoint</h3>
<p>The Fourier viewpoint is one of the most profound connections in linear algebra: the idea that complex signals, data, or functions can be decomposed into sums of simpler, orthogonal waves. Instead of describing information in its raw form (time, space, or coordinates), we express it in terms of frequencies. This perspective reshapes how we analyze, compress, and understand information across mathematics, physics, and engineering.</p>
<section id="fourier-series-the-basic-idea" class="level4">
<h4 class="anchored" data-anchor-id="fourier-series-the-basic-idea">Fourier Series: The Basic Idea</h4>
<p>Suppose we have a periodic function <span class="math inline">\(f(x)\)</span> defined on <span class="math inline">\([-\pi, \pi]\)</span>. The Fourier series expresses <span class="math inline">\(f(x)\)</span> as:</p>
<p><span class="math display">\[
f(x) = a_0 + \sum_{n=1}^\infty \left( a_n \cos(nx) + b_n \sin(nx) \right).
\]</span></p>
<ul>
<li><p>The coefficients <span class="math inline">\(a_n, b_n\)</span> are found using inner products with sine and cosine functions.</p></li>
<li><p>Each sine and cosine is orthogonal to the others under the inner product</p>
<p><span class="math display">\[
\langle f, g \rangle = \int_{-\pi}^\pi f(x) g(x) \, dx.
\]</span></p></li>
</ul>
<p>Thus, Fourier series is nothing more than expanding a function in an orthonormal basis of trigonometric functions.</p>
</section>
<section id="fourier-transform-from-time-to-frequency" class="level4">
<h4 class="anchored" data-anchor-id="fourier-transform-from-time-to-frequency">Fourier Transform: From Time to Frequency</h4>
<p>For non-periodic signals, the Fourier transform generalizes this expansion. For a function <span class="math inline">\(f(t)\)</span>,</p>
<p><span class="math display">\[
\hat{f}(\omega) = \int_{-\infty}^\infty f(t) e^{-i \omega t} dt
\]</span></p>
<p>transforms it into frequency space. The inverse transform reconstructs <span class="math inline">\(f(t)\)</span> from its frequencies.</p>
<p>This is again an inner product viewpoint: the exponential functions <span class="math inline">\(e^{i \omega t}\)</span> act as orthogonal basis functions on <span class="math inline">\(\mathbb{R}\)</span>.</p>
</section>
<section id="orthogonality-of-waves" class="level4">
<h4 class="anchored" data-anchor-id="orthogonality-of-waves">Orthogonality of Waves</h4>
<p>The trigonometric functions <span class="math inline">\(\{\cos(nx), \sin(nx)\}\)</span> and the complex exponentials <span class="math inline">\(\{e^{i\omega t}\}\)</span> form orthogonal families.</p>
<ul>
<li>Two different sine waves have zero inner product over a full period.</li>
<li>Likewise, exponentials with different frequencies are orthogonal.</li>
</ul>
<p>This is exactly like orthogonal vectors in <span class="math inline">\(\mathbb{R}^n\)</span>, except here the space is infinite-dimensional.</p>
</section>
<section id="discrete-fourier-transform-dft" class="level4">
<h4 class="anchored" data-anchor-id="discrete-fourier-transform-dft">Discrete Fourier Transform (DFT)</h4>
<p>In computational settings, we don’t work with infinite integrals but with finite data. The DFT expresses an <span class="math inline">\(n\)</span>-dimensional vector <span class="math inline">\(x = (x_0, \dots, x_{n-1})\)</span> as a linear combination of orthogonal complex exponentials:</p>
<p><span class="math display">\[
X_k = \sum_{j=0}^{n-1} x_j e^{-2\pi i jk / n}, \quad k=0,\dots,n-1.
\]</span></p>
<p>This is simply a change of basis: from the standard basis (time domain) to the Fourier basis (frequency domain).</p>
<p>The Fast Fourier Transform (FFT) computes this in <span class="math inline">\(O(n \log n)\)</span> time, making Fourier analysis practical at scale.</p>
</section>
<section id="geometric-meaning-23" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-23">Geometric Meaning</h4>
<ul>
<li>In the time domain, data is expressed as a sequence of raw values.</li>
<li>In the frequency domain, data is expressed as amplitudes of orthogonal waves.</li>
<li>The Fourier viewpoint is just a rotation into a new orthogonal coordinate system, exactly like diagonalizing a matrix or changing basis.</li>
</ul>
</section>
<section id="applications-41" class="level4">
<h4 class="anchored" data-anchor-id="applications-41">Applications</h4>
<ol type="1">
<li>Signal Processing: Filtering unwanted noise corresponds to removing high-frequency components.</li>
<li>Image Compression: JPEG uses Fourier-like transforms (cosine transforms) to compress images.</li>
<li>Data Analysis: Identifying cycles and periodic patterns in time series.</li>
<li>Physics: Quantum states are represented in both position and momentum bases, linked by Fourier transform.</li>
<li>Partial Differential Equations: Solutions are simplified by moving to frequency space, where derivatives become multipliers.</li>
</ol>
</section>
<section id="why-it-matters-75" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-75">Why It Matters</h4>
<ul>
<li>Fourier methods turn difficult problems into simpler ones: convolution becomes multiplication, differentiation becomes scaling.</li>
<li>They provide a universal language for analyzing periodicity, oscillation, and wave phenomena.</li>
<li>They are linear algebra at heart: orthogonal expansions in special bases.</li>
</ul>
</section>
<section id="try-it-yourself-78" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-78">Try It Yourself</h4>
<ol type="1">
<li>Compute the Fourier series coefficients for <span class="math inline">\(f(x) = x\)</span> on <span class="math inline">\([-\pi,\pi]\)</span>.</li>
<li>For the sequence <span class="math inline">\((1,0,0,0)\)</span>, compute the 4-point DFT and interpret the result.</li>
<li>Show that <span class="math inline">\(\int_{-\pi}^\pi \sin(mx)\cos(nx) dx = 0\)</span>.</li>
<li>Challenge: Prove that the Fourier basis <span class="math inline">\(\{e^{i2\pi k t}\}_{k=0}^{n-1}\)</span> is orthonormal in <span class="math inline">\(\mathbb{C}^n\)</span>.</li>
</ol>
<p>The Fourier viewpoint reveals that every signal, no matter how complex, can be seen as a combination of simple, orthogonal waves. It is a perfect marriage of geometry, algebra, and analysis, and one of the most important ideas in modern mathematics.</p>
</section>
</section>
<section id="polynomial-and-multifeature-least-squares" class="level3">
<h3 class="anchored" data-anchor-id="polynomial-and-multifeature-least-squares">80. Polynomial and Multifeature Least Squares</h3>
<p>Least squares problems become especially powerful when extended to fitting polynomials or handling multiple features at once. Instead of a single straight line through data, we can fit curves of higher degree or surfaces in higher dimensions. These generalizations lie at the heart of regression, data analysis, and scientific modeling.</p>
<section id="from-line-to-polynomial" class="level4">
<h4 class="anchored" data-anchor-id="from-line-to-polynomial">From Line to Polynomial</h4>
<p>The simplest least squares model is a straight line:</p>
<p><span class="math display">\[
y \approx \beta_0 + \beta_1 x.
\]</span></p>
<p>But many relationships are nonlinear. Polynomial least squares generalizes the model to:</p>
<p><span class="math display">\[
y \approx \beta_0 + \beta_1 x + \beta_2 x^2 + \dots + \beta_d x^d.
\]</span></p>
<p>Here, each power of <span class="math inline">\(x\)</span> is treated as a new feature. The problem reduces to ordinary least squares on the design matrix:</p>
<p><span class="math display">\[
A = \begin{bmatrix}
1 &amp; x_1 &amp; x_1^2 &amp; \dots &amp; x_1^d \\
1 &amp; x_2 &amp; x_2^2 &amp; \dots &amp; x_2^d \\
\vdots &amp; \vdots &amp; \vdots &amp; &amp; \vdots \\
1 &amp; x_n &amp; x_n^2 &amp; \dots &amp; x_n^d
\end{bmatrix},
\quad
\beta = \begin{bmatrix}\beta_0 \\ \beta_1 \\ \vdots \\ \beta_d\end{bmatrix}.
\]</span></p>
<p>The least squares solution minimizes <span class="math inline">\(\|A\beta - y\|\)</span>.</p>
</section>
<section id="multiple-features" class="level4">
<h4 class="anchored" data-anchor-id="multiple-features">Multiple Features</h4>
<p>When data involves several predictors, we extend the model to:</p>
<p><span class="math display">\[
y \approx \beta_0 + \beta_1 x_1 + \beta_2 x_2 + \dots + \beta_p x_p.
\]</span></p>
<p>In matrix form:</p>
<p><span class="math display">\[
y \approx A\beta,
\]</span></p>
<p>where <span class="math inline">\(A\)</span> is the design matrix with columns corresponding to features (including a column of ones for the intercept).</p>
<p>The least squares solution is still given by the normal equations:</p>
<p><span class="math display">\[
\hat{\beta} = (A^T A)^{-1} A^T y,
\]</span></p>
<p>or more stably by QR or SVD factorizations.</p>
</section>
<section id="example-polynomial-fit" class="level4">
<h4 class="anchored" data-anchor-id="example-polynomial-fit">Example: Polynomial Fit</h4>
<p>Suppose we have data points <span class="math inline">\((1,1), (2,2.2), (3,2.9), (4,4.1)\)</span>. Fitting a quadratic model <span class="math inline">\(y \approx \beta_0 + \beta_1 x + \beta_2 x^2\)</span>:</p>
<ol type="1">
<li><p>Construct design matrix:</p>
<p><span class="math display">\[
A = \begin{bmatrix}
1 &amp; 1 &amp; 1 \\
1 &amp; 2 &amp; 4 \\
1 &amp; 3 &amp; 9 \\
1 &amp; 4 &amp; 16
\end{bmatrix}.
\]</span></p></li>
<li><p>Solve least squares problem <span class="math inline">\(\min \|A\beta - y\|\)</span>.</p></li>
<li><p>The result gives coefficients <span class="math inline">\(\beta_0, \beta_1, \beta_2\)</span> that best approximate the curve.</p></li>
</ol>
<p>The same process works for higher-degree polynomials or multiple features.</p>
</section>
<section id="geometric-meaning-24" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-24">Geometric Meaning</h4>
<ul>
<li>In polynomial least squares, the feature space expands: instead of points on a line, data lives in a higher-dimensional feature space <span class="math inline">\((1, x, x^2, \dots, x^d)\)</span>.</li>
<li>In multifeature least squares, the column space of <span class="math inline">\(A\)</span> spans all possible linear combinations of features.</li>
<li>The least squares solution projects the observed output vector <span class="math inline">\(y\)</span> onto this subspace.</li>
</ul>
<p>Thus, whether polynomial or multifeature, the geometry is the same: projection onto the model space.</p>
</section>
<section id="practical-challenges" class="level4">
<h4 class="anchored" data-anchor-id="practical-challenges">Practical Challenges</h4>
<ol type="1">
<li>Overfitting: Higher-degree polynomials fit noise, not just signal.</li>
<li>Multicollinearity: Features may be correlated, making <span class="math inline">\(A^T A\)</span> nearly singular.</li>
<li>Scaling: Features with different magnitudes should be normalized.</li>
<li>Regularization: Adding penalty terms (ridge or LASSO) stabilizes the solution.</li>
</ol>
</section>
<section id="applications-42" class="level4">
<h4 class="anchored" data-anchor-id="applications-42">Applications</h4>
<ol type="1">
<li>Regression in Statistics: Extending linear regression to handle multiple predictors or polynomial terms.</li>
<li>Machine Learning: Basis expansion for feature engineering (before neural nets, this was the standard).</li>
<li>Engineering: Curve fitting for calibration, modeling, and prediction.</li>
<li>Economics: Forecasting models with many variables (inflation, interest rates, spending).</li>
<li>Physics and Chemistry: Polynomial regression to model experimental data.</li>
</ol>
</section>
<section id="why-it-matters-76" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-76">Why It Matters</h4>
<ul>
<li>Polynomial least squares captures curvature in data.</li>
<li>Multifeature least squares allows multiple predictors to explain outcomes.</li>
<li>Both generalizations turn linear algebra into a practical modeling tool across science and society.</li>
</ul>
</section>
<section id="try-it-yourself-79" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-79">Try It Yourself</h4>
<ol type="1">
<li>Fit a quadratic curve through points <span class="math inline">\((0,1), (1,2), (2,5), (3,10)\)</span>. Compare to a straight-line fit.</li>
<li>Construct a multifeature design matrix for predicting exam scores based on hours studied, sleep, and prior grades.</li>
<li>Show that polynomial regression is just linear regression on transformed features.</li>
<li>Challenge: Derive the bias–variance tradeoff in polynomial least squares-why higher degrees increase variance.</li>
</ol>
<p>Polynomial and multifeature least squares extend the reach of linear algebra from straight lines to complex patterns, giving us a universal framework for modeling relationships in data.</p>
</section>
<section id="closing-7" class="level4">
<h4 class="anchored" data-anchor-id="closing-7">Closing</h4>
<pre><code>Closest lines are drawn,
errors fall away to rest,
angles guard the truth.</code></pre>
</section>
</section>
</section>
<section id="chapter-9.-svd-pca-and-conditioning" class="level2">
<h2 class="anchored" data-anchor-id="chapter-9.-svd-pca-and-conditioning">Chapter 9. SVD, PCA, and conditioning</h2>
<section id="opening-7" class="level3">
<h3 class="anchored" data-anchor-id="opening-7">Opening</h3>
<pre><code>Closest lines are drawn,
errors fall away to rest,
angles guard the truth.</code></pre>
</section>
<section id="singular-values-and-svd" class="level3">
<h3 class="anchored" data-anchor-id="singular-values-and-svd">81. Singular Values and SVD</h3>
<p>The Singular Value Decomposition (SVD) is one of the most powerful tools in linear algebra. It generalizes eigen-decomposition, works for all rectangular matrices (not just square ones), and provides deep insights into geometry, computation, and data analysis. At its core, the SVD tells us that every matrix can be factored into three pieces: rotations/reflections, scaling, and rotations/reflections again.</p>
<section id="definition-of-svd" class="level4">
<h4 class="anchored" data-anchor-id="definition-of-svd">Definition of SVD</h4>
<p>For any real <span class="math inline">\(m \times n\)</span> matrix <span class="math inline">\(A\)</span>, the SVD is:</p>
<p><span class="math display">\[
A = U \Sigma V^T,
\]</span></p>
<p>where:</p>
<ul>
<li><span class="math inline">\(U\)</span> is an <span class="math inline">\(m \times m\)</span> orthogonal matrix (columns = left singular vectors).</li>
<li><span class="math inline">\(\Sigma\)</span> is an <span class="math inline">\(m \times n\)</span> diagonal matrix with nonnegative entries <span class="math inline">\(\sigma_1 \geq \sigma_2 \geq \dots \geq 0\)</span> (singular values).</li>
<li><span class="math inline">\(V\)</span> is an <span class="math inline">\(n \times n\)</span> orthogonal matrix (columns = right singular vectors).</li>
</ul>
<p>Even if <span class="math inline">\(A\)</span> is rectangular or not diagonalizable, this factorization always exists.</p>
</section>
<section id="geometric-meaning-25" class="level4">
<h4 class="anchored" data-anchor-id="geometric-meaning-25">Geometric Meaning</h4>
<p>The SVD describes how <span class="math inline">\(A\)</span> transforms space:</p>
<ol type="1">
<li>First rotation/reflection: Multiply by <span class="math inline">\(V^T\)</span> to rotate or reflect coordinates into the right singular vector basis.</li>
<li>Scaling: Multiply by <span class="math inline">\(\Sigma\)</span>, stretching/shrinking each axis by a singular value.</li>
<li>Second rotation/reflection: Multiply by <span class="math inline">\(U\)</span> to reorient into the output space.</li>
</ol>
<p>Thus, <span class="math inline">\(A\)</span> acts as a rotation, followed by scaling, followed by another rotation.</p>
</section>
<section id="singular-values" class="level4">
<h4 class="anchored" data-anchor-id="singular-values">Singular Values</h4>
<ul>
<li>The singular values <span class="math inline">\(\sigma_i\)</span> are the square roots of the eigenvalues of <span class="math inline">\(A^T A\)</span>.</li>
<li>They measure how much <span class="math inline">\(A\)</span> stretches space in particular directions.</li>
<li>The largest singular value <span class="math inline">\(\sigma_1\)</span> is the operator norm of <span class="math inline">\(A\)</span>: the maximum stretch factor.</li>
<li>If some singular values are zero, they correspond to directions collapsed by <span class="math inline">\(A\)</span>.</li>
</ul>
</section>
<section id="example-in-mathbbr2-6" class="level4">
<h4 class="anchored" data-anchor-id="example-in-mathbbr2-6">Example in <span class="math inline">\(\mathbb{R}^2\)</span></h4>
<p>Let</p>
<p><span class="math display">\[
A = \begin{bmatrix} 3 &amp; 1 \\ 0 &amp; 2 \end{bmatrix}.
\]</span></p>
<ol type="1">
<li>Compute <span class="math inline">\(A^T A = \begin{bmatrix} 9 &amp; 3 \\ 3 &amp; 5 \end{bmatrix}\)</span>.</li>
<li>Find its eigenvalues: <span class="math inline">\(\lambda_1, \lambda_2 = 10 \pm \sqrt{10}\)</span>.</li>
<li>Singular values: <span class="math inline">\(\sigma_i = \sqrt{\lambda_i}\)</span>.</li>
<li>The corresponding eigenvectors form the right singular vectors <span class="math inline">\(V\)</span>.</li>
<li>Left singular vectors <span class="math inline">\(U\)</span> are obtained by <span class="math inline">\(U = AV/\Sigma\)</span>.</li>
</ol>
<p>This decomposition reveals how <span class="math inline">\(A\)</span> reshapes circles into ellipses.</p>
</section>
<section id="links-to-eigen-decomposition" class="level4">
<h4 class="anchored" data-anchor-id="links-to-eigen-decomposition">Links to Eigen-Decomposition</h4>
<ul>
<li>Eigen-decomposition works only for square, diagonalizable matrices.</li>
<li>SVD works for all matrices, square or rectangular, diagonalizable or not.</li>
<li>Instead of eigenvalues (which may be complex or negative), we get singular values (always real and nonnegative).</li>
<li>Eigenvectors can fail to exist in a full basis; singular vectors always form orthonormal bases.</li>
</ul>
</section>
<section id="applications-43" class="level4">
<h4 class="anchored" data-anchor-id="applications-43">Applications</h4>
<ol type="1">
<li>Data Compression: Truncate small singular values to approximate matrices with fewer dimensions (used in JPEG).</li>
<li>Principal Component Analysis (PCA): SVD on centered data finds principal components, directions of maximum variance.</li>
<li>Least Squares Problems: SVD provides stable solutions, even for ill-conditioned or singular systems.</li>
<li>Noise Filtering: Discard small singular values to remove noise in signals and images.</li>
<li>Numerical Stability: SVD helps diagnose conditioning-how sensitive solutions are to input errors.</li>
</ol>
</section>
<section id="why-it-matters-77" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-77">Why It Matters</h4>
<ul>
<li>SVD is the “Swiss army knife” of linear algebra: versatile, always applicable, and rich in interpretation.</li>
<li>It provides geometric, algebraic, and computational clarity.</li>
<li>It is indispensable for modern applications in machine learning, statistics, engineering, and physics.</li>
</ul>
</section>
<section id="try-it-yourself-80" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-80">Try It Yourself</h4>
<ol type="1">
<li><p>Compute the SVD of</p>
<p><span class="math display">\[
A = \begin{bmatrix}1 &amp; 0 \\ 0 &amp; 2 \\ 0 &amp; 0\end{bmatrix}.
\]</span></p>
<p>Interpret the scaling and rotations.</p></li>
<li><p>Show that for any vector <span class="math inline">\(x\)</span>, <span class="math inline">\(\|Ax\| \leq \sigma_1 \|x\|\)</span>.</p></li>
<li><p>Use SVD to approximate the matrix</p>
<p><span class="math display">\[
\begin{bmatrix}1 &amp; 1 &amp; 1 \\ 1 &amp; 1 &amp; 1 \\ 1 &amp; 1 &amp; 1\end{bmatrix}
\]</span></p>
<p>with rank 1.</p></li>
<li><p>Challenge: Prove that the Frobenius norm of <span class="math inline">\(A\)</span> is the square root of the sum of squares of its singular values.</p></li>
</ol>
<p>The singular value decomposition is universal: every matrix can be dissected into rotations and scalings, revealing its structure and enabling powerful techniques across mathematics and applied sciences.</p>
</section>
</section>
<section id="geometry-of-svd" class="level3">
<h3 class="anchored" data-anchor-id="geometry-of-svd">82. Geometry of SVD</h3>
<p>The Singular Value Decomposition (SVD) is not just an algebraic factorization-it has a precise geometric meaning. It explains exactly how any linear transformation reshapes space: stretching, rotating, compressing, and possibly collapsing dimensions. Understanding this geometry turns SVD from a formal tool into an intuitive picture of what matrices do.</p>
<section id="transformation-of-the-unit-sphere" class="level4">
<h4 class="anchored" data-anchor-id="transformation-of-the-unit-sphere">Transformation of the Unit Sphere</h4>
<p>Take the unit sphere (or circle, in 2D) in the input space. When we apply a matrix <span class="math inline">\(A\)</span>:</p>
<ul>
<li>The sphere is transformed into an ellipsoid.</li>
<li>The axes of this ellipsoid correspond to the right singular vectors <span class="math inline">\(v_i\)</span>.</li>
<li>The lengths of the axes are the singular values <span class="math inline">\(\sigma_i\)</span>.</li>
<li>The directions of the axes in the output space are the left singular vectors <span class="math inline">\(u_i\)</span>.</li>
</ul>
<p>Thus, SVD tells us:</p>
<p><span class="math display">\[
A v_i = \sigma_i u_i.
\]</span></p>
<p>Every matrix maps orthogonal basis directions into orthogonal ellipsoid axes, scaled by singular values.</p>
</section>
<section id="step-by-step-geometry" class="level4">
<h4 class="anchored" data-anchor-id="step-by-step-geometry">Step-by-Step Geometry</h4>
<p>The decomposition <span class="math inline">\(A = U \Sigma V^T\)</span> can be read geometrically:</p>
<ol type="1">
<li>Rotate/reflect by <span class="math inline">\(V^T\)</span>: Align input coordinates with the “principal directions” of <span class="math inline">\(A\)</span>.</li>
<li>Scale by <span class="math inline">\(\Sigma\)</span>: Stretch or compress each axis by its singular value. Some singular values may be zero, flattening dimensions.</li>
<li>Rotate/reflect by <span class="math inline">\(U\)</span>: Reorient the scaled axes into the output space.</li>
</ol>
<p>This process is universal: no matter how irregular a matrix seems, it always reshapes space by rotation → scaling → rotation.</p>
</section>
<section id="d-example" class="level4">
<h4 class="anchored" data-anchor-id="d-example">2D Example</h4>
<p>Take</p>
<p><span class="math display">\[
A = \begin{bmatrix}3 &amp; 1 \\ 0 &amp; 2\end{bmatrix}.
\]</span></p>
<ul>
<li>A circle in <span class="math inline">\(\mathbb{R}^2\)</span> is mapped into an ellipse.</li>
<li>The ellipse’s major and minor axes align with the right singular vectors of <span class="math inline">\(A\)</span>.</li>
<li>Their lengths equal the singular values.</li>
<li>The ellipse itself is then oriented in the output plane according to the left singular vectors.</li>
</ul>
<p>This makes SVD the perfect tool for visualizing how <span class="math inline">\(A\)</span> “distorts” geometry.</p>
</section>
<section id="stretching-and-rank" class="level4">
<h4 class="anchored" data-anchor-id="stretching-and-rank">Stretching and Rank</h4>
<ul>
<li>If all singular values are positive, the ellipsoid has full dimension (no collapse).</li>
<li>If some singular values are zero, <span class="math inline">\(A\)</span> flattens the sphere along certain directions, lowering the rank.</li>
<li>The rank of <span class="math inline">\(A\)</span> equals the number of nonzero singular values.</li>
</ul>
<p>Thus, rank-deficient matrices literally squash space into lower dimensions.</p>
</section>
<section id="distance-and-energy-preservation" class="level4">
<h4 class="anchored" data-anchor-id="distance-and-energy-preservation">Distance and Energy Preservation</h4>
<ul>
<li>The largest singular value <span class="math inline">\(\sigma_1\)</span> is how much <span class="math inline">\(A\)</span> can stretch a vector.</li>
<li>The smallest nonzero singular value <span class="math inline">\(\sigma_r\)</span> (where <span class="math inline">\(r = \text{rank}(A)\)</span>) measures how much the matrix compresses.</li>
<li>The condition number <span class="math inline">\(\kappa(A) = \sigma_1 / \sigma_r\)</span> measures distortion: small values mean nearly spherical stretching, large values mean extreme elongation.</li>
</ul>
</section>
<section id="applications-of-the-geometry" class="level4">
<h4 class="anchored" data-anchor-id="applications-of-the-geometry">Applications of the Geometry</h4>
<ol type="1">
<li>Data Compression: Keeping only the largest singular values keeps the “major axes” of variation.</li>
<li>PCA: Data is analyzed along orthogonal axes of greatest variance (singular vectors).</li>
<li>Numerical Analysis: Geometry of SVD shows why ill-conditioned systems amplify errors-because some directions are squashed almost flat.</li>
<li>Signal Processing: Elliptical distortions correspond to filtering out certain frequency components.</li>
<li>Machine Learning: Dimensionality reduction is essentially projecting data onto the largest singular directions.</li>
</ol>
</section>
<section id="why-it-matters-78" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-78">Why It Matters</h4>
<ul>
<li>SVD transforms algebraic equations into geometric pictures.</li>
<li>It reveals exactly how matrices warp space, offering intuition behind abstract operations.</li>
<li>By interpreting ellipses, singular values, and orthogonal vectors, we gain visual clarity for problems in data, physics, and computation.</li>
</ul>
</section>
<section id="try-it-yourself-81" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-81">Try It Yourself</h4>
<ol type="1">
<li><p>Draw the unit circle in <span class="math inline">\(\mathbb{R}^2\)</span>, apply the matrix</p>
<p><span class="math display">\[
A = \begin{bmatrix}2 &amp; 0 \\ 1 &amp; 3\end{bmatrix},
\]</span></p>
<p>and sketch the resulting ellipse. Identify its axes and lengths.</p></li>
<li><p>Verify numerically that <span class="math inline">\(Av_i = \sigma_i u_i\)</span> for computed singular vectors and singular values.</p></li>
<li><p>For a rank-1 matrix, sketch how the unit circle collapses to a line segment.</p></li>
<li><p>Challenge: Prove that the set of vectors with maximum stretch under <span class="math inline">\(A\)</span> are precisely the first right singular vectors.</p></li>
</ol>
<p>The geometry of SVD gives us a universal lens: every linear transformation is a controlled distortion of space, built from orthogonal rotations and directional scalings.</p>
</section>
</section>
<section id="relation-to-eigen-decompositions" class="level3">
<h3 class="anchored" data-anchor-id="relation-to-eigen-decompositions">83. Relation to Eigen-Decompositions</h3>
<p>The Singular Value Decomposition (SVD) is often introduced as something entirely new, but it is deeply tied to eigen-decomposition. In fact, singular values and singular vectors emerge from the eigen-decomposition of certain symmetric matrices constructed from <span class="math inline">\(A\)</span>. Understanding this connection shows why SVD always exists, why singular values are nonnegative, and how it generalizes eigen-analysis to all matrices, even rectangular ones.</p>
<section id="eigen-decomposition-recap" class="level4">
<h4 class="anchored" data-anchor-id="eigen-decomposition-recap">Eigen-Decomposition Recap</h4>
<p>For a square matrix <span class="math inline">\(M \in \mathbb{R}^{n \times n}\)</span>, an eigen-decomposition is:</p>
<p><span class="math display">\[
M = X \Lambda X^{-1},
\]</span></p>
<p>where <span class="math inline">\(\Lambda\)</span> is a diagonal matrix of eigenvalues and the columns of <span class="math inline">\(X\)</span> are eigenvectors.</p>
<p>However:</p>
<ul>
<li>Not all matrices are diagonalizable.</li>
<li>Eigenvalues may be complex.</li>
<li>Rectangular matrices don’t have eigenvalues at all.</li>
</ul>
<p>This is where SVD provides a universal framework.</p>
</section>
<section id="from-at-a-to-singular-values" class="level4">
<h4 class="anchored" data-anchor-id="from-at-a-to-singular-values">From <span class="math inline">\(A^T A\)</span> to Singular Values</h4>
<p>For any <span class="math inline">\(m \times n\)</span> matrix <span class="math inline">\(A\)</span>:</p>
<ol type="1">
<li><p>Consider the symmetric, positive semidefinite matrix <span class="math inline">\(A^T A \in \mathbb{R}^{n \times n}\)</span>.</p>
<ul>
<li>Symmetry ensures all eigenvalues are real.</li>
<li>Positive semidefiniteness ensures they are nonnegative.</li>
</ul></li>
<li><p>The eigenvalues of <span class="math inline">\(A^T A\)</span> are squares of the singular values of <span class="math inline">\(A\)</span>:</p>
<p><span class="math display">\[
\lambda_i(A^T A) = \sigma_i^2.
\]</span></p></li>
<li><p>The eigenvectors of <span class="math inline">\(A^T A\)</span> are the right singular vectors <span class="math inline">\(v_i\)</span>.</p></li>
<li><p>Similarly, for <span class="math inline">\(AA^T\)</span>, eigenvalues are the same <span class="math inline">\(\sigma_i^2\)</span>, and eigenvectors are the left singular vectors <span class="math inline">\(u_i\)</span>.</p></li>
</ol>
<p>Thus:</p>
<p><span class="math display">\[
Av_i = \sigma_i u_i, \quad A^T u_i = \sigma_i v_i.
\]</span></p>
<p>This pair of relationships binds eigen-decomposition and SVD together.</p>
</section>
<section id="why-eigen-decomposition-is-not-enough" class="level4">
<h4 class="anchored" data-anchor-id="why-eigen-decomposition-is-not-enough">Why Eigen-Decomposition Is Not Enough</h4>
<ul>
<li>Eigen-decomposition requires a square matrix. SVD works for rectangular matrices.</li>
<li>Eigenvalues can be negative or complex; singular values are always real and nonnegative.</li>
<li>Eigenvectors may not exist as a complete basis; singular vectors always form orthonormal bases.</li>
</ul>
<p>In short, SVD provides the robustness that eigen-decomposition lacks.</p>
</section>
<section id="example-4" class="level4">
<h4 class="anchored" data-anchor-id="example-4">Example</h4>
<p>Let</p>
<p><span class="math display">\[
A = \begin{bmatrix}3 &amp; 0 \\ 4 &amp; 0 \\ 0 &amp; 5\end{bmatrix}.
\]</span></p>
<ol type="1">
<li><p>Compute <span class="math inline">\(A^T A = \begin{bmatrix}25 &amp; 0 \\ 0 &amp; 25\end{bmatrix}\)</span>.</p>
<ul>
<li>Eigenvalues: <span class="math inline">\(25, 25\)</span>.</li>
<li>Singular values: <span class="math inline">\(\sigma_1 = \sigma_2 = 5\)</span>.</li>
</ul></li>
<li><p>Right singular vectors are eigenvectors of <span class="math inline">\(A^T A\)</span>. Here, they form the standard basis.</p></li>
<li><p>Left singular vectors come from <span class="math inline">\(Av_i / \sigma_i\)</span>.</p></li>
</ol>
<p>So the geometry of SVD is fully encoded in eigen-analysis of <span class="math inline">\(A^T A\)</span> and <span class="math inline">\(AA^T\)</span>.</p>
</section>
<section id="geometric-picture-1" class="level4">
<h4 class="anchored" data-anchor-id="geometric-picture-1">Geometric Picture</h4>
<ul>
<li>Eigenvectors of <span class="math inline">\(A^T A\)</span> describe directions in input space where <span class="math inline">\(A\)</span> stretches without mixing directions.</li>
<li>Eigenvectors of <span class="math inline">\(AA^T\)</span> describe the corresponding directions in output space.</li>
<li>Singular values tell us how much stretching occurs.</li>
</ul>
<p>Thus, SVD is essentially eigen-decomposition in disguise-but applied to the right symmetric companions.</p>
</section>
<section id="applications-of-the-connection" class="level4">
<h4 class="anchored" data-anchor-id="applications-of-the-connection">Applications of the Connection</h4>
<ol type="1">
<li>PCA: Data covariance matrix <span class="math inline">\(X^T X\)</span> uses eigen-decomposition, but PCA is implemented with SVD directly.</li>
<li>Numerical Methods: Algorithms for SVD rely on eigen-analysis of <span class="math inline">\(A^T A\)</span>.</li>
<li>Stability Analysis: The relationship ensures singular values are reliable measures of conditioning.</li>
<li>Signal Processing: Power in signals (variance) is explained by eigenvalues of covariance, which connect to singular values.</li>
<li>Machine Learning: Kernel PCA and related methods depend on this link to handle nonlinear features.</li>
</ol>
</section>
<section id="why-it-matters-79" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-79">Why It Matters</h4>
<ul>
<li>SVD explains every matrix transformation in terms of orthogonal bases and scalings.</li>
<li>Its relationship with eigen-decomposition ensures that SVD is not an alien tool, but a generalization.</li>
<li>The eigenview shows why SVD is guaranteed to exist and why singular values are always real and nonnegative.</li>
</ul>
</section>
<section id="try-it-yourself-82" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-82">Try It Yourself</h4>
<ol type="1">
<li><p>Prove that if <span class="math inline">\(v\)</span> is an eigenvector of <span class="math inline">\(A^T A\)</span> with eigenvalue <span class="math inline">\(\lambda\)</span>, then <span class="math inline">\(Av\)</span> is either zero or a left singular vector of <span class="math inline">\(A\)</span> with singular value <span class="math inline">\(\sqrt{\lambda}\)</span>.</p></li>
<li><p>For the matrix</p>
<p><span class="math display">\[
A = \begin{bmatrix}1 &amp; 2 \\ 2 &amp; 1\end{bmatrix},
\]</span></p>
<p>compute both eigen-decomposition and SVD. Compare the results.</p></li>
<li><p>Show that <span class="math inline">\(A^T A\)</span> and <span class="math inline">\(AA^T\)</span> always share the same nonzero eigenvalues.</p></li>
<li><p>Challenge: Explain why an orthogonal diagonalization of <span class="math inline">\(A^T A\)</span> is enough to guarantee existence of the full SVD of <span class="math inline">\(A\)</span>.</p></li>
</ol>
<p>The relationship between SVD and eigen-decomposition unifies two of linear algebra’s deepest ideas: every matrix transformation is built from eigen-geometry, stretched into a form that always exists and always makes sense.</p>
</section>
</section>
<section id="low-rank-approximation-best-small-models" class="level3">
<h3 class="anchored" data-anchor-id="low-rank-approximation-best-small-models">84. Low-Rank Approximation (Best Small Models)</h3>
<p>A central idea in data analysis, scientific computing, and machine learning is that many datasets or matrices are far more complicated in raw form than they truly need to be. Much of the apparent complexity hides redundancy, noise, or low-dimensional patterns. Low-rank approximation is the process of compressing a large, complicated matrix into a smaller, simpler version that preserves the most important information. This concept, grounded in the Singular Value Decomposition (SVD), lies at the heart of dimensionality reduction, recommender systems, and modern AI.</p>
<section id="the-general-problem" class="level4">
<h4 class="anchored" data-anchor-id="the-general-problem">The General Problem</h4>
<p>Suppose we have a matrix <span class="math inline">\(A \in \mathbb{R}^{m \times n}\)</span>, perhaps representing:</p>
<ul>
<li>An image, with rows as pixels and columns as color channels.</li>
<li>A ratings table, with rows as users and columns as movies.</li>
<li>A word embedding matrix, with rows as words and columns as features.</li>
</ul>
<p>Often, <span class="math inline">\(A\)</span> is very large but highly structured. The question is:</p>
<p><em>Can we find a smaller matrix <span class="math inline">\(B\)</span> of rank <span class="math inline">\(k\)</span> (where <span class="math inline">\(k \ll \min(m, n)\)</span>) that approximates <span class="math inline">\(A\)</span> well?</em></p>
</section>
<section id="rank-and-complexity" class="level4">
<h4 class="anchored" data-anchor-id="rank-and-complexity">Rank and Complexity</h4>
<p>The rank of a matrix is the number of independent directions it encodes. High rank means complexity; low rank means redundancy.</p>
<ul>
<li>A rank-1 matrix can be written as an outer product of two vectors: <span class="math inline">\(uv^T\)</span>.</li>
<li>A rank-<span class="math inline">\(k\)</span> matrix is a sum of <span class="math inline">\(k\)</span> such outer products.</li>
<li>Limiting rank controls how much structure we allow the approximation to capture.</li>
</ul>
</section>
<section id="the-svd-solution" class="level4">
<h4 class="anchored" data-anchor-id="the-svd-solution">The SVD Solution</h4>
<p>The SVD provides a natural decomposition:</p>
<p><span class="math display">\[
A = U \Sigma V^T,
\]</span></p>
<p>where singular values <span class="math inline">\(\sigma_1 \geq \sigma_2 \geq \cdots \geq \sigma_r\)</span> measure importance.</p>
<p>To approximate <span class="math inline">\(A\)</span> with rank <span class="math inline">\(k\)</span>:</p>
<p><span class="math display">\[
A_k = U_k \Sigma_k V_k^T,
\]</span></p>
<p>where we keep only the top <span class="math inline">\(k\)</span> singular values and vectors.</p>
<p>This is not just a heuristic: it is the Eckart–Young theorem:</p>
<blockquote class="blockquote">
<p>Among all rank-<span class="math inline">\(k\)</span> matrices, <span class="math inline">\(A_k\)</span> minimizes the error <span class="math inline">\(\|A - B\|\)</span> (both in Frobenius and spectral norm).</p>
</blockquote>
<p>Thus, SVD provides the <em>best possible</em> low-rank approximation.</p>
</section>
<section id="geometric-intuition-6" class="level4">
<h4 class="anchored" data-anchor-id="geometric-intuition-6">Geometric Intuition</h4>
<ul>
<li>Each singular value <span class="math inline">\(\sigma_i\)</span> measures how strongly <span class="math inline">\(A\)</span> stretches in the direction of singular vector <span class="math inline">\(v_i\)</span>.</li>
<li>Keeping the top <span class="math inline">\(k\)</span> singular values means keeping the most important stretches and ignoring weaker directions.</li>
<li>The approximation captures the “essence” of <span class="math inline">\(A\)</span> while discarding small, noisy, or redundant effects.</li>
</ul>
</section>
<section id="examples-5" class="level4">
<h4 class="anchored" data-anchor-id="examples-5">Examples</h4>
<ol type="1">
<li>Images A grayscale image can be stored as a matrix of pixel intensities. Using SVD, one can compress it by keeping only the largest singular values:</li>
</ol>
<ul>
<li><span class="math inline">\(k = 10\)</span>: blurry but recognizable image.</li>
<li><span class="math inline">\(k = 50\)</span>: much sharper, yet storage cost is far less than full.</li>
<li><span class="math inline">\(k = 200\)</span>: nearly indistinguishable from the original.</li>
</ul>
<p>This is practical image compression: fewer numbers, same perception.</p>
<ol start="2" type="1">
<li><p>Recommender Systems Consider a user–movie rating matrix. Although it may be huge, the true patterns (genre preferences, popularity trends) live in a low-dimensional subspace. A rank-<span class="math inline">\(k\)</span> approximation captures these patterns, predicting missing ratings by filling in the structure.</p></li>
<li><p>Natural Language Processing (NLP) Word embeddings often arise from co-occurrence matrices. Low-rank approximation via SVD extracts semantic structure, enabling words like “king,” “queen,” and “crown” to cluster together.</p></li>
</ol>
</section>
<section id="error-and-trade-offs" class="level4">
<h4 class="anchored" data-anchor-id="error-and-trade-offs">Error and Trade-Offs</h4>
<ul>
<li>Error decay: If singular values drop quickly, small <span class="math inline">\(k\)</span> gives a great approximation. If they decay slowly, more terms are needed.</li>
<li>Energy preserved: The squared singular values <span class="math inline">\(\sigma_i^2\)</span> represent variance captured. Keeping the first <span class="math inline">\(k\)</span> terms preserves most of the “energy.”</li>
<li>Balance: Too low rank = oversimplification (loss of structure). Too high rank = no compression.</li>
</ul>
</section>
<section id="practical-computation" class="level4">
<h4 class="anchored" data-anchor-id="practical-computation">Practical Computation</h4>
<p>For very large matrices, full SVD is expensive (<span class="math inline">\(O(mn^2)\)</span> for <span class="math inline">\(m \geq n\)</span>). Alternatives include:</p>
<ul>
<li>Truncated SVD algorithms (Lanczos, randomized methods).</li>
<li>Iterative methods that compute only the top <span class="math inline">\(k\)</span> singular values.</li>
<li>Incremental approaches that update low-rank models as new data arrives.</li>
</ul>
<p>These are vital in modern data science, where datasets often have millions of entries.</p>
</section>
<section id="analogy" class="level4">
<h4 class="anchored" data-anchor-id="analogy">Analogy</h4>
<ul>
<li>Music playlist: Imagine a playlist with hundreds of songs, but most are variations on a few themes. A low-rank approximation is like keeping only the core melodies while discarding repetitive riffs.</li>
<li>Photograph compression: Keeping only the brightest and most important strokes of light, while ignoring faint and irrelevant details.</li>
<li>Book summary: Instead of the full text, you read the essential plot points. That’s low-rank approximation.</li>
</ul>
</section>
<section id="why-it-matters-80" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-80">Why It Matters</h4>
<ul>
<li>Reveals hidden structure in high-dimensional data.</li>
<li>Reduces storage and computational cost.</li>
<li>Filters noise while preserving the signal.</li>
<li>Provides the foundation for PCA, recommender systems, and dimensionality reduction.</li>
</ul>
</section>
<section id="try-it-yourself-83" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-83">Try It Yourself</h4>
<ol type="1">
<li>Take a small <span class="math inline">\(5 \times 5\)</span> random matrix. Compute its SVD. Construct the best rank-1 approximation. Compare to the original.</li>
<li>Download a grayscale image (e.g., <span class="math inline">\(256 \times 256\)</span>). Reconstruct it with 10, 50, and 100 singular values. Visually compare.</li>
<li>Prove the Eckart–Young theorem for the spectral norm: why can no other rank-<span class="math inline">\(k\)</span> approximation do better than truncated SVD?</li>
<li>For a dataset with many features, compute PCA and explain why it is equivalent to finding a low-rank approximation.</li>
</ol>
<p>Low-rank approximation shows how linear algebra captures the essence of complexity: most of what matters lives in a small number of dimensions. The art is in finding and using them effectively.</p>
</section>
</section>
<section id="principal-component-analysis-variance-and-directions" class="level3">
<h3 class="anchored" data-anchor-id="principal-component-analysis-variance-and-directions">85. Principal Component Analysis (Variance and Directions)</h3>
<p>Principal Component Analysis (PCA) is one of the most widely used techniques in statistics, data analysis, and machine learning. It provides a method to reduce the dimensionality of a dataset while retaining as much important information as possible. The central insight is that data often varies more strongly in some directions than others, and by focusing on those directions we can summarize the dataset with fewer dimensions, less noise, and more interpretability.</p>
<section id="the-basic-question" class="level4">
<h4 class="anchored" data-anchor-id="the-basic-question">The Basic Question</h4>
<p>Suppose we have data points in high-dimensional space, say <span class="math inline">\(x_1, x_2, \dots, x_m \in \mathbb{R}^n\)</span>. Each point might be:</p>
<ul>
<li>A face image flattened into thousands of pixels.</li>
<li>A customer’s shopping history across hundreds of products.</li>
<li>A gene expression profile across thousands of genes.</li>
</ul>
<p>Storing and working with all features directly is expensive, and many features may be redundant or correlated. PCA asks:</p>
<p><em>Can we re-express this data in a smaller set of directions that capture the most variability?</em></p>
</section>
<section id="variance-as-information" class="level4">
<h4 class="anchored" data-anchor-id="variance-as-information">Variance as Information</h4>
<p>The guiding principle of PCA is variance.</p>
<ul>
<li>Variance measures how spread out the data is along a direction.</li>
<li>High variance directions capture meaningful structure (e.g., different facial expressions, major spending habits).</li>
<li>Low variance directions often correspond to noise or unimportant fluctuations.</li>
</ul>
<p>Thus, PCA searches for the directions (called principal components) along which the variance of the data is maximized.</p>
</section>
<section id="centering-and-covariance" class="level4">
<h4 class="anchored" data-anchor-id="centering-and-covariance">Centering and Covariance</h4>
<p>To begin, we center the data by subtracting the mean vector:</p>
<p><span class="math display">\[
X_c = X - \mathbf{1}\mu^T,
\]</span></p>
<p>where <span class="math inline">\(\mu\)</span> is the average of all data points.</p>
<p>The covariance matrix is then:</p>
<p><span class="math display">\[
C = \frac{1}{m} X_c^T X_c.
\]</span></p>
<ul>
<li>The diagonal entries measure variance of each feature.</li>
<li>Off-diagonal entries measure how features vary together.</li>
</ul>
<p>Finding principal components is equivalent to finding the eigenvectors of this covariance matrix.</p>
</section>
<section id="the-eigenview" class="level4">
<h4 class="anchored" data-anchor-id="the-eigenview">The Eigenview</h4>
<ol type="1">
<li>The eigenvectors of <span class="math inline">\(C\)</span> are the directions (principal components).</li>
<li>The corresponding eigenvalues tell us how much variance lies along each component.</li>
<li>Sorting eigenvalues from largest to smallest gives the most informative to least informative directions.</li>
</ol>
<p>If we keep the top <span class="math inline">\(k\)</span> eigenvectors, we project data into a <span class="math inline">\(k\)</span>-dimensional subspace that preserves most variance.</p>
</section>
<section id="the-svd-view" class="level4">
<h4 class="anchored" data-anchor-id="the-svd-view">The SVD View</h4>
<p>Another perspective uses the Singular Value Decomposition (SVD):</p>
<p><span class="math display">\[
X_c = U \Sigma V^T.
\]</span></p>
<ul>
<li>Columns of <span class="math inline">\(V\)</span> are the principal directions.</li>
<li>Singular values squared (<span class="math inline">\(\sigma_i^2\)</span>) correspond to eigenvalues of the covariance matrix.</li>
<li>Projecting onto the first <span class="math inline">\(k\)</span> columns of <span class="math inline">\(V\)</span> gives the reduced representation.</li>
</ul>
<p>This makes PCA and SVD essentially the same computation.</p>
</section>
<section id="a-simple-example-1" class="level4">
<h4 class="anchored" data-anchor-id="a-simple-example-1">A Simple Example</h4>
<p>Imagine we measure height and weight of 1000 people. Plotting them shows a strong correlation: taller people are often heavier. The cloud of points stretches along a diagonal.</p>
<ul>
<li>PCA’s first component is this diagonal line: the direction of maximum variance.</li>
<li>The second component is perpendicular, capturing the much smaller differences (like people of equal height but slightly different weights).</li>
<li>Keeping only the first component reduces two features into one while retaining most of the information.</li>
</ul>
</section>
<section id="geometric-picture-2" class="level4">
<h4 class="anchored" data-anchor-id="geometric-picture-2">Geometric Picture</h4>
<ul>
<li>PCA rotates the coordinate system so that axes align with directions of greatest variance.</li>
<li>Projecting onto the top <span class="math inline">\(k\)</span> components flattens the data into a lower-dimensional space, like flattening a tilted pancake onto its broadest plane.</li>
</ul>
</section>
<section id="applications-44" class="level4">
<h4 class="anchored" data-anchor-id="applications-44">Applications</h4>
<ol type="1">
<li>Data Compression: Reduce storage by keeping only leading components (e.g., compressing images).</li>
<li>Noise Reduction: Small-variance directions often correspond to measurement noise; discarding them yields cleaner data.</li>
<li>Visualization: Reducing data to 2D or 3D for scatterplots helps us see clusters and patterns.</li>
<li>Preprocessing in Machine Learning: Many models train faster and generalize better on PCA-transformed data.</li>
<li>Genomics and Biology: PCA finds major axes of variation across thousands of genes.</li>
<li>Finance: PCA summarizes correlated movements of stocks into a few principal “factors.”</li>
</ol>
</section>
<section id="trade-offs-and-limitations" class="level4">
<h4 class="anchored" data-anchor-id="trade-offs-and-limitations">Trade-Offs and Limitations</h4>
<ul>
<li>Interpretability: Principal components are linear combinations of original features, sometimes hard to explain in plain terms.</li>
<li>Linearity: PCA only captures linear relationships; nonlinear methods (like kernel PCA, t-SNE, or UMAP) may be better for curved manifolds.</li>
<li>Scaling: Features must be normalized properly; otherwise, PCA might overemphasize units with large raw variance.</li>
<li>Global Method: PCA captures overall variance, not local structures (e.g., small clusters within the data).</li>
</ul>
</section>
<section id="mathematical-guarantees" class="level4">
<h4 class="anchored" data-anchor-id="mathematical-guarantees">Mathematical Guarantees</h4>
<p>PCA has an optimality guarantee:</p>
<ul>
<li>Among all <span class="math inline">\(k\)</span>-dimensional linear subspaces, the PCA subspace minimizes the reconstruction error (squared Euclidean distance between data and its projection).</li>
<li>This is essentially the low-rank approximation theorem seen earlier, applied to covariance matrices.</li>
</ul>
</section>
<section id="why-it-matters-81" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-81">Why It Matters</h4>
<p>PCA shows how linear algebra transforms raw data into insight. By focusing on variance, it provides a principled way to filter noise, compress information, and reveal hidden patterns. It is simple, computationally efficient, and foundational-almost every modern data pipeline uses PCA, explicitly or implicitly.</p>
</section>
<section id="try-it-yourself-84" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-84">Try It Yourself</h4>
<ol type="1">
<li>Take a dataset of two correlated features (like height and weight). Compute the covariance matrix, eigenvectors, and project onto the first component. Visualize before and after.</li>
<li>For a grayscale image stored as a matrix, flatten it into vectors and apply PCA. How many components are needed to reconstruct it with 90% accuracy?</li>
<li>Use PCA on the famous Iris dataset (4 features). Plot the data in 2D using the first two components. Notice how species separate in this reduced space.</li>
<li>Prove that the first principal component is the unit vector <span class="math inline">\(v\)</span> that maximizes <span class="math inline">\(\|X_c v\|^2\)</span>.</li>
</ol>
<p>PCA distills complexity into clarity: it tells us not just where the data is, but where it <em>really goes</em>.</p>
</section>
</section>
<section id="pseudoinverse-moorepenrose-and-solving-ill-posed-systems" class="level3">
<h3 class="anchored" data-anchor-id="pseudoinverse-moorepenrose-and-solving-ill-posed-systems">86. Pseudoinverse (Moore–Penrose) and Solving Ill-Posed Systems</h3>
<p>In linear algebra, the inverse of a matrix is a powerful tool: if <span class="math inline">\(A\)</span> is invertible, then solving <span class="math inline">\(Ax = b\)</span> is as simple as <span class="math inline">\(x = A^{-1}b\)</span>. But what happens when <span class="math inline">\(A\)</span> is not square, or not invertible? In practice, this is the norm: many problems involve rectangular matrices (more equations than unknowns, or more unknowns than equations), or square matrices that are singular. The Moore–Penrose pseudoinverse, usually denoted <span class="math inline">\(A^+\)</span>, generalizes the idea of an inverse to all matrices, providing a systematic way to find solutions-or best approximations-when ordinary inversion fails.</p>
<section id="why-ordinary-inverses-fail" class="level4">
<h4 class="anchored" data-anchor-id="why-ordinary-inverses-fail">Why Ordinary Inverses Fail</h4>
<ul>
<li>Non-square matrices: If <span class="math inline">\(A\)</span> is <span class="math inline">\(m \times n\)</span> with <span class="math inline">\(m \neq n\)</span>, no standard inverse exists.</li>
<li>Singular matrices: Even if <span class="math inline">\(A\)</span> is square, if <span class="math inline">\(\det(A) = 0\)</span>, it has no inverse.</li>
<li>Ill-posed problems: In real-world data, exact solutions may not exist (inconsistent systems) or may not be unique (underdetermined systems).</li>
</ul>
<p>Despite these obstacles, we still want a systematic way to solve or approximate <span class="math inline">\(Ax = b\)</span>.</p>
</section>
<section id="definition-of-the-pseudoinverse" class="level4">
<h4 class="anchored" data-anchor-id="definition-of-the-pseudoinverse">Definition of the Pseudoinverse</h4>
<p>The Moore–Penrose pseudoinverse <span class="math inline">\(A^+\)</span> is defined as the unique matrix that satisfies four properties:</p>
<ol type="1">
<li><span class="math inline">\(AA^+A = A\)</span>.</li>
<li><span class="math inline">\(A^+AA^+ = A^+\)</span>.</li>
<li><span class="math inline">\((AA^+)^T = AA^+\)</span>.</li>
<li><span class="math inline">\((A^+A)^T = A^+A\)</span>.</li>
</ol>
<p>These conditions ensure <span class="math inline">\(A^+\)</span> acts as an “inverse” in the broadest consistent sense.</p>
</section>
<section id="constructing-the-pseudoinverse-with-svd" class="level4">
<h4 class="anchored" data-anchor-id="constructing-the-pseudoinverse-with-svd">Constructing the Pseudoinverse with SVD</h4>
<p>Given the SVD of <span class="math inline">\(A\)</span>:</p>
<p><span class="math display">\[
A = U \Sigma V^T,
\]</span></p>
<p>where <span class="math inline">\(\Sigma\)</span> is diagonal with singular values <span class="math inline">\(\sigma_1, \dots, \sigma_r\)</span>, the pseudoinverse is:</p>
<p><span class="math display">\[
A^+ = V \Sigma^+ U^T,
\]</span></p>
<p>where <span class="math inline">\(\Sigma^+\)</span> is formed by inverting nonzero singular values and transposing the matrix. Specifically:</p>
<ul>
<li>If <span class="math inline">\(\sigma_i \neq 0\)</span>, replace it with <span class="math inline">\(1/\sigma_i\)</span>.</li>
<li>If <span class="math inline">\(\sigma_i = 0\)</span>, leave it as 0.</li>
</ul>
<p>This definition works for all matrices, square or rectangular.</p>
</section>
<section id="solving-linear-systems-with-a" class="level4">
<h4 class="anchored" data-anchor-id="solving-linear-systems-with-a">Solving Linear Systems with <span class="math inline">\(A^+\)</span></h4>
<ol type="1">
<li><p>Overdetermined systems (<span class="math inline">\(m &gt; n\)</span>, more equations than unknowns):</p>
<ul>
<li><p>Often no exact solution exists.</p></li>
<li><p>The pseudoinverse gives the least-squares solution:</p>
<p><span class="math display">\[
x = A^+ b,
\]</span></p>
<p>which minimizes <span class="math inline">\(\|Ax - b\|\)</span>.</p></li>
</ul></li>
<li><p>Underdetermined systems (<span class="math inline">\(m &lt; n\)</span>, more unknowns than equations):</p>
<ul>
<li><p>Infinitely many solutions exist.</p></li>
<li><p>The pseudoinverse chooses the solution with the smallest norm:</p>
<p><span class="math display">\[
x = A^+ b,
\]</span></p>
<p>which minimizes <span class="math inline">\(\|x\|\)</span> among all solutions.</p></li>
</ul></li>
<li><p>Square but singular systems:</p>
<ul>
<li>Some solutions exist, but not uniquely.</li>
<li>The pseudoinverse again picks the least-norm solution.</li>
</ul></li>
</ol>
</section>
<section id="example-1-overdetermined" class="level4">
<h4 class="anchored" data-anchor-id="example-1-overdetermined">Example 1: Overdetermined</h4>
<p>Suppose we want to solve:</p>
<p><span class="math display">\[
\begin{bmatrix}1 &amp; 1 \\ 1 &amp; -1 \\ 1 &amp; 0\end{bmatrix} x = \begin{bmatrix}2 \\ 0 \\ 1\end{bmatrix}.
\]</span></p>
<p>This <span class="math inline">\(3 \times 2\)</span> system has no exact solution. Using the pseudoinverse, we obtain the least-squares solution that best fits all three equations simultaneously.</p>
</section>
<section id="example-2-underdetermined" class="level4">
<h4 class="anchored" data-anchor-id="example-2-underdetermined">Example 2: Underdetermined</h4>
<p>For</p>
<p><span class="math display">\[
\begin{bmatrix}1 &amp; 0 &amp; 0 \\ 0 &amp; 1 &amp; 0\end{bmatrix} x = \begin{bmatrix}3 \\ 4\end{bmatrix},
\]</span></p>
<p>the system has infinitely many solutions because <span class="math inline">\(x_3\)</span> is free. The pseudoinverse gives:</p>
<p><span class="math display">\[
x = \begin{bmatrix}3 \\ 4 \\ 0\end{bmatrix},
\]</span></p>
<p>choosing the solution with minimum norm.</p>
</section>
<section id="geometric-interpretation-16" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-16">Geometric Interpretation</h4>
<ul>
<li>The pseudoinverse acts like projecting onto subspaces.</li>
<li>For overdetermined systems, it projects <span class="math inline">\(b\)</span> onto the column space of <span class="math inline">\(A\)</span>, then finds the closest <span class="math inline">\(x\)</span>.</li>
<li>For underdetermined systems, it picks the point in the solution space closest to the origin.</li>
</ul>
<p>So <span class="math inline">\(A^+\)</span> embodies the principle of “best possible inverse” under the circumstances.</p>
</section>
<section id="applications-45" class="level4">
<h4 class="anchored" data-anchor-id="applications-45">Applications</h4>
<ol type="1">
<li>Least-Squares Regression: Solving <span class="math inline">\(\min_x \|Ax - b\|^2\)</span> via <span class="math inline">\(A^+\)</span>.</li>
<li>Signal Processing: Reconstructing signals from incomplete or noisy data.</li>
<li>Control Theory: Designing inputs when exact control is impossible.</li>
<li>Machine Learning: Training models with non-invertible design matrices.</li>
<li>Statistics: Computing generalized inverses of covariance matrices.</li>
</ol>
</section>
<section id="limitations" class="level4">
<h4 class="anchored" data-anchor-id="limitations">Limitations</h4>
<ul>
<li>Sensitive to very small singular values: numerical instability may occur.</li>
<li>Regularization (like ridge regression) is often preferred in noisy settings.</li>
<li>Computationally expensive for very large matrices, though truncated SVD can help.</li>
</ul>
</section>
<section id="why-it-matters-82" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-82">Why It Matters</h4>
<p>The pseudoinverse is a unifying idea: it handles inconsistent, underdetermined, or singular problems with one formula. It ensures we always have a principled answer, even when classical algebra says “no solution” or “infinitely many solutions.” In real data analysis, almost every problem is ill-posed to some degree, making the pseudoinverse a practical cornerstone of modern applied linear algebra.</p>
</section>
<section id="try-it-yourself-85" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-85">Try It Yourself</h4>
<ol type="1">
<li>Compute the pseudoinverse of a simple <span class="math inline">\(2 \times 2\)</span> singular matrix by hand using SVD.</li>
<li>Solve both an overdetermined (<span class="math inline">\(3 \times 2\)</span>) and underdetermined (<span class="math inline">\(2 \times 3\)</span>) system using <span class="math inline">\(A^+\)</span>. Compare with intuitive expectations.</li>
<li>Explore what happens numerically when singular values are very small. Try truncating them-this connects to regularization.</li>
</ol>
<p>The Moore–Penrose pseudoinverse shows that even when linear systems are “broken,” linear algebra still provides a systematic way forward.</p>
</section>
</section>
<section id="conditioning-and-sensitivity-how-errors-amplify" class="level3">
<h3 class="anchored" data-anchor-id="conditioning-and-sensitivity-how-errors-amplify">87. Conditioning and Sensitivity (How Errors Amplify)</h3>
<p>Linear algebra is not only about exact solutions-it is also about how <em>stable</em> those solutions are when data is perturbed. In real-world applications, every dataset contains noise: measurement errors in physics experiments, rounding errors in financial computations, or floating-point precision limits in numerical software. Conditioning is the study of how sensitive the solution of a problem is to small changes in input. A well-conditioned problem reacts gently to perturbations; an ill-conditioned one amplifies errors dramatically.</p>
<section id="the-basic-idea" class="level4">
<h4 class="anchored" data-anchor-id="the-basic-idea">The Basic Idea</h4>
<p>Suppose we solve the linear system:</p>
<p><span class="math display">\[
Ax = b.
\]</span></p>
<p>Now imagine we slightly change <span class="math inline">\(b\)</span> to <span class="math inline">\(b + \delta b\)</span>. The new solution is <span class="math inline">\(x + \delta x\)</span>.</p>
<ul>
<li>If <span class="math inline">\(\|\delta x\|\)</span> is about the same size as <span class="math inline">\(\|\delta b\|\)</span>, the problem is well-conditioned.</li>
<li>If <span class="math inline">\(\|\delta x\|\)</span> is much larger, the problem is ill-conditioned.</li>
</ul>
<p>Conditioning measures this amplification factor.</p>
</section>
<section id="condition-number" class="level4">
<h4 class="anchored" data-anchor-id="condition-number">Condition Number</h4>
<p>The central tool is the condition number of a matrix <span class="math inline">\(A\)</span>:</p>
<p><span class="math display">\[
\kappa(A) = \|A\| \cdot \|A^{-1}\|,
\]</span></p>
<p>where <span class="math inline">\(\|\cdot\|\)</span> is a matrix norm (often the 2-norm).</p>
<ul>
<li>If <span class="math inline">\(\kappa(A)\)</span> is close to 1, the problem is well-conditioned.</li>
<li>If <span class="math inline">\(\kappa(A)\)</span> is large (say, <span class="math inline">\(10^6\)</span> or higher), the problem is ill-conditioned.</li>
</ul>
<p>Interpretation:</p>
<ul>
<li><span class="math inline">\(\kappa(A)\)</span> estimates the maximum relative error in the solution compared to the relative error in the data.</li>
<li>In practical terms, every digit of accuracy in <span class="math inline">\(b\)</span> may be lost in <span class="math inline">\(x\)</span> if <span class="math inline">\(\kappa(A)\)</span> is too large.</li>
</ul>
</section>
<section id="singular-values-and-conditioning" class="level4">
<h4 class="anchored" data-anchor-id="singular-values-and-conditioning">Singular Values and Conditioning</h4>
<p>Condition number in 2-norm can be expressed using singular values:</p>
<p><span class="math display">\[
\kappa(A) = \frac{\sigma_{\max}}{\sigma_{\min}},
\]</span></p>
<p>where <span class="math inline">\(\sigma_{\max}\)</span> and <span class="math inline">\(\sigma_{\min}\)</span> are the largest and smallest singular values of <span class="math inline">\(A\)</span>.</p>
<ul>
<li>If the smallest singular value is tiny compared to the largest, <span class="math inline">\(A\)</span> nearly collapses some directions, making inversion unstable.</li>
<li>This explains why nearly singular matrices are so problematic in numerical computation.</li>
</ul>
</section>
<section id="example-1-a-stable-system" class="level4">
<h4 class="anchored" data-anchor-id="example-1-a-stable-system">Example 1: A Stable System</h4>
<p><span class="math display">\[
A = \begin{bmatrix}2 &amp; 0 \\ 0 &amp; 3\end{bmatrix}.
\]</span></p>
<p>Here, <span class="math inline">\(\sigma_{\max} = 3, \sigma_{\min} = 2\)</span>. So <span class="math inline">\(\kappa(A) = 3/2 = 1.5\)</span>. Very well-conditioned: small changes in input produce small changes in output.</p>
</section>
<section id="example-2-an-ill-conditioned-system" class="level4">
<h4 class="anchored" data-anchor-id="example-2-an-ill-conditioned-system">Example 2: An Ill-Conditioned System</h4>
<p><span class="math display">\[
A = \begin{bmatrix}1 &amp; 1 \\ 1 &amp; 1.0001\end{bmatrix}.
\]</span></p>
<p>The determinant is very small, so the system is nearly singular.</p>
<ul>
<li>One singular value is about 2.0.</li>
<li>The other is about 0.0001.</li>
<li>Condition number: <span class="math inline">\(\kappa(A) \approx 20000\)</span>.</li>
</ul>
<p>This means even tiny changes in <span class="math inline">\(b\)</span> can wildly change <span class="math inline">\(x\)</span>.</p>
</section>
<section id="geometric-intuition-7" class="level4">
<h4 class="anchored" data-anchor-id="geometric-intuition-7">Geometric Intuition</h4>
<p>A matrix transforms a unit sphere into an ellipse.</p>
<ul>
<li>The longest axis of the ellipse = <span class="math inline">\(\sigma_{\max}\)</span>.</li>
<li>The shortest axis = <span class="math inline">\(\sigma_{\min}\)</span>.</li>
<li>The ratio <span class="math inline">\(\sigma_{\max} / \sigma_{\min}\)</span> shows how stretched the transformation is.</li>
</ul>
<p>If the ellipse is nearly flat, directions aligned with the short axis almost vanish, and recovering them is highly unstable.</p>
</section>
<section id="why-conditioning-matters-in-computation" class="level4">
<h4 class="anchored" data-anchor-id="why-conditioning-matters-in-computation">Why Conditioning Matters in Computation</h4>
<ol type="1">
<li>Numerical Precision: Computers store numbers with limited precision (floating-point). An ill-conditioned system magnifies rounding errors, leading to unreliable results.</li>
<li>Regression: In statistics, highly correlated features make the design matrix ill-conditioned, destabilizing coefficient estimates.</li>
<li>Machine Learning: Ill-conditioning leads to unstable training, exploding or vanishing gradients.</li>
<li>Engineering: Control systems based on ill-conditioned models may be hypersensitive to measurement errors.</li>
</ol>
</section>
<section id="techniques-for-handling-ill-conditioning" class="level4">
<h4 class="anchored" data-anchor-id="techniques-for-handling-ill-conditioning">Techniques for Handling Ill-Conditioning</h4>
<ul>
<li>Regularization: Add a penalty term, like ridge regression (<span class="math inline">\(\lambda I\)</span>), to stabilize inversion.</li>
<li>Truncated SVD: Ignore tiny singular values that only amplify noise.</li>
<li>Scaling and Preconditioning: Rescale data or multiply by a well-chosen matrix to improve conditioning.</li>
<li>Avoiding Explicit Inverses: Use factorizations (LU, QR, SVD) rather than computing <span class="math inline">\(A^{-1}\)</span>.</li>
</ul>
</section>
<section id="connection-to-previous-topics" class="level4">
<h4 class="anchored" data-anchor-id="connection-to-previous-topics">Connection to Previous Topics</h4>
<ul>
<li>Pseudoinverse: Ill-conditioning is visible when singular values approach zero, making <span class="math inline">\(A^+\)</span> unstable.</li>
<li>Low-rank approximation: Truncating small singular values both compresses data and improves conditioning.</li>
<li>PCA: Discarding low-variance components is essentially a conditioning improvement step.</li>
</ul>
</section>
<section id="why-it-matters-83" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-83">Why It Matters</h4>
<p>Conditioning bridges abstract algebra and numerical reality. Linear algebra promises solutions, but conditioning tells us whether those solutions are trustworthy. Without it, one might misinterpret noise as signal, or lose all accuracy in computations that look fine on paper.</p>
</section>
<section id="try-it-yourself-86" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-86">Try It Yourself</h4>
<ol type="1">
<li>Compute the condition number of <span class="math inline">\(\begin{bmatrix}1 &amp; 1 \\ 1 &amp; 1.0001\end{bmatrix}\)</span>. Solve for <span class="math inline">\(x\)</span> in <span class="math inline">\(Ax = b\)</span> for several slightly different <span class="math inline">\(b\)</span>. Watch how solutions swing dramatically.</li>
<li>Take a dataset with nearly collinear features. Compute the condition number of its covariance matrix. Relate this to instability in regression coefficients.</li>
<li>Simulate numerical errors: Add random noise of size <span class="math inline">\(10^{-6}\)</span> to an ill-conditioned system and observe solution errors.</li>
<li>Prove that <span class="math inline">\(\kappa(A) \geq 1\)</span> always holds.</li>
</ol>
<p>Conditioning reveals the hidden fragility of problems. It warns us when algebra says “solution exists” but computation whispers “don’t trust it.”</p>
</section>
</section>
<section id="matrix-norms-and-singular-values-measuring-size-properly" class="level3">
<h3 class="anchored" data-anchor-id="matrix-norms-and-singular-values-measuring-size-properly">88. Matrix Norms and Singular Values (Measuring Size Properly)</h3>
<p>In linear algebra, we often need to measure the “size” of a matrix. For vectors, this is straightforward: the length (norm) tells us how big the vector is. But for matrices, the question is more subtle: do we measure size by entries, by how much the matrix stretches vectors, or by some invariant property? Different contexts demand different answers, and matrix norms-closely tied to singular values-provide the framework for doing so.</p>
<section id="why-measure-the-size-of-a-matrix" class="level4">
<h4 class="anchored" data-anchor-id="why-measure-the-size-of-a-matrix">Why Measure the Size of a Matrix?</h4>
<ol type="1">
<li>Stability: To know how much error a matrix might amplify.</li>
<li>Conditioning: The ratio of largest to smallest stretching.</li>
<li>Optimization: Many algorithms minimize some matrix norm.</li>
<li>Data analysis: Norms measure complexity or energy of data.</li>
</ol>
<p>Without norms, we cannot compare matrices, analyze sensitivity, or judge approximation quality.</p>
</section>
<section id="matrix-norms-from-vector-norms" class="level4">
<h4 class="anchored" data-anchor-id="matrix-norms-from-vector-norms">Matrix Norms from Vector Norms</h4>
<p>A natural way to define a matrix norm is to ask: <em>How much does this matrix stretch vectors?</em></p>
<p>Formally, for a given vector norm <span class="math inline">\(\|\cdot\|\)</span>:</p>
<p><span class="math display">\[
\|A\| = \max_{x \neq 0} \frac{\|Ax\|}{\|x\|}.
\]</span></p>
<p>This is called the induced matrix norm.</p>
</section>
<section id="the-2-norm-and-singular-values" class="level4">
<h4 class="anchored" data-anchor-id="the-2-norm-and-singular-values">The 2-Norm and Singular Values</h4>
<p>When we use the Euclidean norm (<span class="math inline">\(\|x\|_2\)</span>) for vectors, the induced matrix norm becomes:</p>
<p><span class="math display">\[
\|A\|_2 = \sigma_{\max}(A),
\]</span></p>
<p>the largest singular value of <span class="math inline">\(A\)</span>.</p>
<ul>
<li>This means the 2-norm measures the <em>maximum stretching factor</em>.</li>
<li>Geometrically: <span class="math inline">\(A\)</span> maps the unit sphere into an ellipse; <span class="math inline">\(\|A\|_2\)</span> is the length of the ellipse’s longest axis.</li>
</ul>
<p>This link makes singular values the natural language for matrix size.</p>
</section>
<section id="other-common-norms" class="level4">
<h4 class="anchored" data-anchor-id="other-common-norms">Other Common Norms</h4>
<ol type="1">
<li>Frobenius Norm</li>
</ol>
<p><span class="math display">\[
\|A\|_F = \sqrt{\sum_{i,j} |a_{ij}|^2}.
\]</span></p>
<ul>
<li><p>Equivalent to the Euclidean length of all entries stacked in one big vector.</p></li>
<li><p>Can also be expressed as:</p>
<p><span class="math display">\[
\|A\|_F^2 = \sum_i \sigma_i^2.
\]</span></p></li>
<li><p>Often used in data science and machine learning because it is easy to compute and differentiable.</p></li>
</ul>
<ol start="2" type="1">
<li>1-Norm</li>
</ol>
<p><span class="math display">\[
\|A\|_1 = \max_j \sum_i |a_{ij}|,
\]</span></p>
<p>the maximum absolute column sum.</p>
<ol start="3" type="1">
<li>Infinity Norm</li>
</ol>
<p><span class="math display">\[
\|A\|_\infty = \max_i \sum_j |a_{ij}|,
\]</span></p>
<p>the maximum absolute row sum.</p>
<p>Both are computationally cheap, useful in numerical analysis.</p>
<ol start="4" type="1">
<li>Nuclear Norm (Trace Norm)</li>
</ol>
<p><span class="math display">\[
\|A\|_* = \sum_i \sigma_i,
\]</span></p>
<p>the sum of singular values.</p>
<ul>
<li>Important in low-rank approximation and machine learning (matrix completion, recommender systems).</li>
</ul>
</section>
<section id="singular-values-as-the-unifying-thread" class="level4">
<h4 class="anchored" data-anchor-id="singular-values-as-the-unifying-thread">Singular Values as the Unifying Thread</h4>
<ul>
<li>Spectral norm (2-norm): maximum singular value.</li>
<li>Frobenius norm: root of the sum of squared singular values.</li>
<li>Nuclear norm: sum of singular values.</li>
</ul>
<p>Thus, norms capture different ways of summarizing singular values: maximum, sum, or energy.</p>
</section>
<section id="example-small-matrix" class="level4">
<h4 class="anchored" data-anchor-id="example-small-matrix">Example: Small Matrix</h4>
<p>Take</p>
<p><span class="math display">\[
A = \begin{bmatrix}3 &amp; 4 \\ 0 &amp; 0\end{bmatrix}.
\]</span></p>
<ul>
<li>Singular values: <span class="math inline">\(\sigma_1 = 5, \sigma_2 = 0\)</span>.</li>
<li><span class="math inline">\(\|A\|_2 = 5\)</span>.</li>
<li><span class="math inline">\(\|A\|_F = \sqrt{3^2 + 4^2} = 5\)</span>.</li>
<li><span class="math inline">\(\|A\|_* = 5\)</span>.</li>
</ul>
<p>Here, different norms coincide, but generally they highlight different aspects of the matrix.</p>
</section>
<section id="geometric-intuition-8" class="level4">
<h4 class="anchored" data-anchor-id="geometric-intuition-8">Geometric Intuition</h4>
<ul>
<li>2-norm: “How much can <span class="math inline">\(A\)</span> stretch a vector?”</li>
<li>Frobenius norm: “What is the overall energy in all entries?”</li>
<li>1-norm / ∞-norm: “What is the heaviest column or row load?”</li>
<li>Nuclear norm: “How much total stretching power does <span class="math inline">\(A\)</span> have?”</li>
</ul>
<p>Each is a lens, giving a different perspective.</p>
</section>
<section id="applications-46" class="level4">
<h4 class="anchored" data-anchor-id="applications-46">Applications</h4>
<ol type="1">
<li>Numerical Stability: Condition number <span class="math inline">\(\kappa(A) = \sigma_{\max}/\sigma_{\min}\)</span> uses the spectral norm.</li>
<li>Machine Learning: Nuclear norm is used for matrix completion (Netflix Prize).</li>
<li>Image Compression: Frobenius norm measures reconstruction error.</li>
<li>Control Theory: 1-norm and ∞-norm bound system responses.</li>
<li>Optimization: Norms serve as penalties or constraints, shaping solutions.</li>
</ol>
</section>
<section id="why-it-matters-84" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-84">Why It Matters</h4>
<p>Matrix norms provide the language to compare, approximate, and control matrices. Singular values ensure that this language is not arbitrary but grounded in geometry. Together, they explain how matrices distort space, how error grows, and how we can measure complexity.</p>
</section>
<section id="try-it-yourself-87" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-87">Try It Yourself</h4>
<ol type="1">
<li>For <span class="math inline">\(A = \begin{bmatrix}1 &amp; 2 \\ 3 &amp; 4\end{bmatrix}\)</span>, compute <span class="math inline">\(\|A\|_1\)</span>, <span class="math inline">\(\|A\|_\infty\)</span>, <span class="math inline">\(\|A\|_F\)</span>, and <span class="math inline">\(\|A\|_2\)</span> (using SVD for the last). Compare.</li>
<li>Prove that <span class="math inline">\(\|A\|_F^2 = \sum \sigma_i^2\)</span>.</li>
<li>Show that <span class="math inline">\(\|A\|_2 \leq \|A\|_F \leq \|A\|_*\)</span>. Interpret geometrically.</li>
<li>Consider a rank-1 matrix <span class="math inline">\(uv^T\)</span>. What are its norms? Which are equal?</li>
</ol>
<p>Matrix norms and singular values are the measuring sticks of linear algebra-they tell us not just how big a matrix is, but how it acts, where it is stable, and when it is fragile.</p>
</section>
</section>
<section id="regularization-ridgetikhonov-to-tame-instability" class="level3">
<h3 class="anchored" data-anchor-id="regularization-ridgetikhonov-to-tame-instability">89. Regularization (Ridge/Tikhonov to Tame Instability)</h3>
<p>When solving linear systems or regression problems, instability often arises because the system is ill-conditioned: tiny errors in data lead to huge swings in the solution. Regularization is the strategy of <em>adding stability</em> by deliberately modifying the problem, sacrificing exactness for robustness. The two most common approaches-ridge regression and Tikhonov regularization-embody this principle.</p>
<section id="the-problem-of-instability" class="level4">
<h4 class="anchored" data-anchor-id="the-problem-of-instability">The Problem of Instability</h4>
<p>Consider the least-squares problem:</p>
<p><span class="math display">\[
\min_x \|Ax - b\|_2^2.
\]</span></p>
<p>If <span class="math inline">\(A\)</span> has nearly dependent columns, or if <span class="math inline">\(\sigma_{\min}(A)\)</span> is very small, then:</p>
<ul>
<li>Solutions are unstable.</li>
<li>Coefficients <span class="math inline">\(x\)</span> can explode in magnitude.</li>
<li>Predictions vary wildly with small changes in <span class="math inline">\(b\)</span>.</li>
</ul>
<p>Regularization modifies the objective so that the solution prefers stability over exactness.</p>
</section>
<section id="ridge-tikhonov-regularization" class="level4">
<h4 class="anchored" data-anchor-id="ridge-tikhonov-regularization">Ridge / Tikhonov Regularization</h4>
<p>The modified problem is:</p>
<p><span class="math display">\[
\min_x \big( \|Ax - b\|_2^2 + \lambda \|x\|_2^2 \big),
\]</span></p>
<p>where <span class="math inline">\(\lambda &gt; 0\)</span> is the regularization parameter.</p>
<ul>
<li>The first term enforces data fit.</li>
<li>The second term penalizes large coefficients, discouraging unstable solutions.</li>
</ul>
<p>This is called ridge regression in statistics and Tikhonov regularization in numerical analysis.</p>
</section>
<section id="the-closed-form-solution" class="level4">
<h4 class="anchored" data-anchor-id="the-closed-form-solution">The Closed-Form Solution</h4>
<p>Expanding the objective and differentiating gives:</p>
<p><span class="math display">\[
x_\lambda = (A^T A + \lambda I)^{-1} A^T b.
\]</span></p>
<p>Key points:</p>
<ul>
<li>The added <span class="math inline">\(\lambda I\)</span> makes the matrix invertible, even if <span class="math inline">\(A^T A\)</span> is singular.</li>
<li>As <span class="math inline">\(\lambda \to 0\)</span>, the solution approaches the ordinary least-squares solution.</li>
<li>As <span class="math inline">\(\lambda \to \infty\)</span>, the solution shrinks toward 0.</li>
</ul>
</section>
<section id="svd-view" class="level4">
<h4 class="anchored" data-anchor-id="svd-view">SVD View</h4>
<p>If <span class="math inline">\(A = U \Sigma V^T\)</span>, then the least-squares solution is:</p>
<p><span class="math display">\[
x = \sum_i \frac{u_i^T b}{\sigma_i} v_i.
\]</span></p>
<p>If <span class="math inline">\(\sigma_i\)</span> is very small, the term <span class="math inline">\(\frac{1}{\sigma_i}\)</span> causes instability.</p>
<p>With regularization:</p>
<p><span class="math display">\[
x_\lambda = \sum_i \frac{\sigma_i}{\sigma_i^2 + \lambda} (u_i^T b) v_i.
\]</span></p>
<ul>
<li>Small singular values (unstable directions) are suppressed.</li>
<li>Large singular values (stable directions) are mostly preserved.</li>
</ul>
<p>This explains why ridge regression stabilizes solutions: it damps noise-amplifying directions.</p>
</section>
<section id="geometric-interpretation-17" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-17">Geometric Interpretation</h4>
<ul>
<li>The unregularized problem fits <span class="math inline">\(b\)</span> exactly in the column space of <span class="math inline">\(A\)</span>.</li>
<li>Regularization tilts the solution toward the origin, shrinking coefficients.</li>
<li>Geometrically, the feasible region (ellipsoid from <span class="math inline">\(Ax\)</span>) intersects with a ball constraint from <span class="math inline">\(\|x\|_2\)</span>. The solution is where these two shapes balance.</li>
</ul>
</section>
<section id="extensions-1" class="level4">
<h4 class="anchored" data-anchor-id="extensions-1">Extensions</h4>
<ol type="1">
<li>Lasso (<span class="math inline">\(\ell_1\)</span> regularization): Replaces <span class="math inline">\(\|x\|_2^2\)</span> with <span class="math inline">\(\|x\|_1\)</span>, encouraging sparse solutions.</li>
<li>Elastic Net: Combines ridge and lasso penalties.</li>
<li>General Tikhonov: Uses <span class="math inline">\(\|Lx\|_2^2\)</span> with some matrix <span class="math inline">\(L\)</span>, tailoring the penalty (e.g., smoothing in signal processing).</li>
<li>Bayesian View: Ridge regression corresponds to placing a Gaussian prior on coefficients.</li>
</ol>
</section>
<section id="applications-47" class="level4">
<h4 class="anchored" data-anchor-id="applications-47">Applications</h4>
<ul>
<li>Machine Learning: Prevents overfitting in regression and classification.</li>
<li>Signal Processing: Suppresses noise when reconstructing signals.</li>
<li>Image Reconstruction: Stabilizes inverse problems like deblurring.</li>
<li>Numerical PDEs: Adds smoothness constraints to solutions.</li>
<li>Econometrics and Finance: Controls instability from highly correlated variables.</li>
</ul>
</section>
<section id="why-it-matters-85" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-85">Why It Matters</h4>
<p>Regularization transforms fragile problems into reliable ones. It acknowledges the reality of noise and finite precision, and instead of chasing impossible exactness, it provides usable, stable answers. In modern data-driven fields, almost every large-scale model relies on regularization for robustness.</p>
</section>
<section id="try-it-yourself-88" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-88">Try It Yourself</h4>
<ol type="1">
<li><p>Solve the system <span class="math inline">\(Ax = b\)</span> where</p>
<p><span class="math display">\[
A = \begin{bmatrix}1 &amp; 1 \\ 1 &amp; 1.0001\end{bmatrix}, \quad b = \begin{bmatrix}2 \\ 2\end{bmatrix}.
\]</span></p>
<p>Compare the unregularized least-squares solution with ridge-regularized solutions for <span class="math inline">\(\lambda = 0.01, 1, 10\)</span>.</p></li>
<li><p>Using the SVD, show how coefficients for small singular values are shrunk.</p></li>
<li><p>In regression with many correlated features, compute coefficient paths as <span class="math inline">\(\lambda\)</span> varies. Observe how they stabilize.</p></li>
<li><p>Explore image denoising: apply ridge regularization to a blurred/noisy image reconstruction problem.</p></li>
</ol>
<p>Regularization shows the wisdom of linear algebra in practice: sometimes the best solution is not the exact one, but the stable one.</p>
</section>
</section>
<section id="rank-revealing-qr-and-practical-diagnostics-what-rank-really-is" class="level3">
<h3 class="anchored" data-anchor-id="rank-revealing-qr-and-practical-diagnostics-what-rank-really-is">90. Rank-Revealing QR and Practical Diagnostics (What Rank Really Is)</h3>
<p>Rank-the number of independent directions in a matrix-is central to linear algebra. It tells us about solvability of systems, redundancy of features, and the dimensionality of data. But in practice, computing rank is not as simple as counting pivots or checking determinants. Real-world data is noisy, nearly dependent, or high-dimensional. Rank-revealing QR (RRQR) factorization and related diagnostics provide stable, practical tools for uncovering rank and structure.</p>
<section id="why-rank-matters" class="level4">
<h4 class="anchored" data-anchor-id="why-rank-matters">Why Rank Matters</h4>
<ul>
<li>Linear systems: Rank determines if a system has a unique solution, infinitely many, or none.</li>
<li>Data science: Rank measures intrinsic dimensionality, guiding dimensionality reduction.</li>
<li>Numerics: Small singular values make effective rank ambiguous-exact vs.&nbsp;numerical rank diverge.</li>
</ul>
<p>Thus, we need reliable algorithms to decide “how many directions matter” in a matrix.</p>
</section>
<section id="exact-rank-vs.-numerical-rank" class="level4">
<h4 class="anchored" data-anchor-id="exact-rank-vs.-numerical-rank">Exact Rank vs.&nbsp;Numerical Rank</h4>
<ul>
<li>Exact rank: Defined over exact arithmetic. A column is independent if it cannot be expressed as a linear combination of others.</li>
<li>Numerical rank: In floating-point computation, tiny singular values cannot be trusted. A threshold <span class="math inline">\(\epsilon\)</span> determines when we treat them as zero.</li>
</ul>
<p>For example, if the smallest singular value of <span class="math inline">\(A\)</span> is <span class="math inline">\(10^{-12}\)</span>, and computations are in double precision (<span class="math inline">\(\sim 10^{-16}\)</span>), we might consider the effective rank smaller than full.</p>
</section>
<section id="the-qr-factorization-recap" class="level4">
<h4 class="anchored" data-anchor-id="the-qr-factorization-recap">The QR Factorization Recap</h4>
<p>The basic QR factorization expresses a matrix <span class="math inline">\(A \in \mathbb{R}^{m \times n}\)</span> as:</p>
<p><span class="math display">\[
A = QR,
\]</span></p>
<p>where:</p>
<ul>
<li><span class="math inline">\(Q\)</span> is orthogonal (<span class="math inline">\(Q^T Q = I\)</span>), preserving lengths.</li>
<li><span class="math inline">\(R\)</span> is upper triangular, holding the “essence” of <span class="math inline">\(A\)</span>.</li>
</ul>
<p>QR is stable, fast, and forms the backbone of many algorithms.</p>
</section>
<section id="rank-revealing-qr-rrqr" class="level4">
<h4 class="anchored" data-anchor-id="rank-revealing-qr-rrqr">Rank-Revealing QR (RRQR)</h4>
<p>RRQR is an enhancement of QR with column pivoting:</p>
<p><span class="math display">\[
A P = Q R,
\]</span></p>
<p>where <span class="math inline">\(P\)</span> is a permutation matrix that reorders columns.</p>
<ul>
<li>The pivoting ensures that the largest independent directions come first.</li>
<li>The diagonal entries of <span class="math inline">\(R\)</span> indicate which columns are significant.</li>
<li>Small values on the diagonal signal dependent (or nearly dependent) directions.</li>
</ul>
<p>In practice, RRQR allows us to approximate rank by examining the decay of <span class="math inline">\(R\)</span>’s diagonal.</p>
</section>
<section id="comparing-rrqr-and-svd" class="level4">
<h4 class="anchored" data-anchor-id="comparing-rrqr-and-svd">Comparing RRQR and SVD</h4>
<ul>
<li>SVD: Gold standard for determining rank; singular values give exact scaling of each direction.</li>
<li>RRQR: Faster and cheaper; sufficient when approximate rank is enough.</li>
<li>Trade-off: SVD is more accurate, RRQR is more efficient.</li>
</ul>
<p>Both are used depending on the balance of precision and cost.</p>
</section>
<section id="example-5" class="level4">
<h4 class="anchored" data-anchor-id="example-5">Example</h4>
<p>Let</p>
<p><span class="math display">\[
A = \begin{bmatrix}1 &amp; 1 &amp; 1 \\ 1 &amp; 1.0001 &amp; 2 \\ 1 &amp; 2 &amp; 3\end{bmatrix}.
\]</span></p>
<ul>
<li>Exact arithmetic: rank = 3.</li>
<li>Numerically: second column is nearly dependent on the first. SVD shows a singular value near zero.</li>
<li>RRQR with pivoting identifies the near-dependence by revealing a tiny diagonal in <span class="math inline">\(R\)</span>.</li>
</ul>
<p>Thus, RRQR “reveals” effective rank without fully computing SVD.</p>
</section>
<section id="practical-diagnostics-for-rank" class="level4">
<h4 class="anchored" data-anchor-id="practical-diagnostics-for-rank">Practical Diagnostics for Rank</h4>
<ol type="1">
<li>Condition Number: A high condition number suggests near-rank-deficiency.</li>
<li>Diagonal of R in RRQR: Monitors independence of columns.</li>
<li>Singular Values in SVD: Most reliable indicator, but expensive.</li>
<li>Determinants/Minors: Useful in theory, unstable in practice.</li>
</ol>
</section>
<section id="applications-48" class="level4">
<h4 class="anchored" data-anchor-id="applications-48">Applications</h4>
<ul>
<li>Data Compression: Identifying effective rank allows truncation.</li>
<li>Regression: Detecting multicollinearity by examining rank of the design matrix.</li>
<li>Control Systems: Rank tests stability and controllability.</li>
<li>Machine Learning: Dimensionality reduction pipelines (e.g., PCA) start with rank estimation.</li>
<li>Signal Processing: Identifying number of underlying sources from mixtures.</li>
</ul>
</section>
<section id="why-it-matters-86" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-86">Why It Matters</h4>
<p>Rank is simple in theory, but elusive in practice. RRQR and related diagnostics bridge the gap between exact mathematics and noisy data. They allow practitioners to say, with stability and confidence: <em>this is how many independent directions really matter.</em></p>
</section>
<section id="try-it-yourself-89" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-89">Try It Yourself</h4>
<ol type="1">
<li>Implement RRQR with column pivoting on a small <span class="math inline">\(5 \times 5\)</span> nearly dependent matrix. Compare estimated rank with SVD.</li>
<li>Explore the relationship between diagonal entries of <span class="math inline">\(R\)</span> and numerical rank.</li>
<li>Construct a dataset with 100 features, where 95 are random noise but 5 are linear combinations. Use RRQR to detect redundancy.</li>
<li>Prove that column pivoting does not change the column space of <span class="math inline">\(A\)</span>, only its numerical stability.</li>
</ol>
<p>Rank-revealing QR shows that linear algebra is not only about exact formulas but also about practical diagnostics-knowing when two directions are truly different and when they are essentially the same.</p>
</section>
<section id="closing-8" class="level4">
<h4 class="anchored" data-anchor-id="closing-8">Closing</h4>
<pre><code>Noise reduced to still,
singular values unfold space,
essence shines within.</code></pre>
</section>
</section>
</section>
<section id="chapter-10.-applications-and-computation" class="level2">
<h2 class="anchored" data-anchor-id="chapter-10.-applications-and-computation">Chapter 10. Applications and computation</h2>
<section id="opening-8" class="level4">
<h4 class="anchored" data-anchor-id="opening-8">Opening</h4>
<pre><code>Worlds in numbers bloom,
graphs and data interlace,
algebra takes flight.</code></pre>
</section>
<section id="d3d-geometry-pipelines-cameras-rotations-and-transforms" class="level3">
<h3 class="anchored" data-anchor-id="d3d-geometry-pipelines-cameras-rotations-and-transforms">91. 2D/3D Geometry Pipelines (Cameras, Rotations, and Transforms)</h3>
<p>Linear algebra is the silent backbone of modern graphics, robotics, and computer vision. Every time an image is rendered on a screen, a camera captures a scene, or a robot arm moves in space, a series of matrix multiplications are transforming points from one coordinate system to another. These geometry pipelines map 3D reality into 2D representations, ensuring that objects appear in the correct position, orientation, and scale.</p>
<section id="the-geometry-of-coordinates" class="level4">
<h4 class="anchored" data-anchor-id="the-geometry-of-coordinates">The Geometry of Coordinates</h4>
<p>A point in 3D space is represented as a column vector:</p>
<p><span class="math display">\[
p = \begin{bmatrix} x \\ y \\ z \end{bmatrix}.
\]</span></p>
<p>But computers often extend this to homogeneous coordinates, embedding the point in 4D:</p>
<p><span class="math display">\[
p_h = \begin{bmatrix} x \\ y \\ z \\ 1 \end{bmatrix}.
\]</span></p>
<p>The extra coordinate allows translations to be represented as matrix multiplications, keeping the entire pipeline consistent: every step is just multiplying by a matrix.</p>
</section>
<section id="transformations-in-2d-and-3d" class="level4">
<h4 class="anchored" data-anchor-id="transformations-in-2d-and-3d">Transformations in 2D and 3D</h4>
<ul>
<li><p>Translation Moves a point by <span class="math inline">\((t_x, t_y, t_z)\)</span>.</p>
<p><span class="math display">\[
T = \begin{bmatrix}
1 &amp; 0 &amp; 0 &amp; t_x \\
0 &amp; 1 &amp; 0 &amp; t_y \\
0 &amp; 0 &amp; 1 &amp; t_z \\
0 &amp; 0 &amp; 0 &amp; 1
\end{bmatrix}.
\]</span></p></li>
<li><p>Scaling Expands or shrinks space along each axis.</p>
<p><span class="math display">\[
S = \begin{bmatrix}
s_x &amp; 0 &amp; 0 &amp; 0 \\
0 &amp; s_y &amp; 0 &amp; 0 \\
0 &amp; 0 &amp; s_z &amp; 0 \\
0 &amp; 0 &amp; 0 &amp; 1
\end{bmatrix}.
\]</span></p></li>
<li><p>Rotation In 3D, rotation around the z-axis is:</p>
<p><span class="math display">\[
R_z(\theta) = \begin{bmatrix}
\cos\theta &amp; -\sin\theta &amp; 0 &amp; 0 \\
\sin\theta &amp; \cos\theta &amp; 0 &amp; 0 \\
0 &amp; 0 &amp; 1 &amp; 0 \\
0 &amp; 0 &amp; 0 &amp; 1
\end{bmatrix}.
\]</span></p>
<p>Similar forms exist for rotations around the x- and y-axes.</p></li>
</ul>
<p>Each transformation is linear (or affine), and chaining them is just multiplying matrices.</p>
</section>
<section id="the-camera-pipeline" class="level4">
<h4 class="anchored" data-anchor-id="the-camera-pipeline">The Camera Pipeline</h4>
<p>Rendering a 3D object to a 2D image follows a sequence of steps, each one a matrix multiplication:</p>
<ol type="1">
<li><p>Model Transform Moves the object from its local coordinates into world coordinates.</p></li>
<li><p>View Transform Puts the camera at the origin and aligns its axes with the world, effectively changing the point of view.</p></li>
<li><p>Projection Transform Projects 3D points into 2D. Two types:</p>
<ul>
<li>Orthographic: parallel projection, no perspective.</li>
<li>Perspective: distant objects appear smaller, closer to human vision.</li>
</ul>
<p>Example of perspective projection:</p>
<p><span class="math display">\[
P = \begin{bmatrix}
f &amp; 0 &amp; 0 &amp; 0 \\
0 &amp; f &amp; 0 &amp; 0 \\
0 &amp; 0 &amp; 1 &amp; 0
\end{bmatrix},
\]</span></p>
<p>where <span class="math inline">\(f\)</span> is focal length.</p></li>
<li><p>Viewport Transform Maps normalized 2D coordinates to screen pixels.</p></li>
</ol>
<p>This sequence-from object to image-is the geometry pipeline.</p>
</section>
<section id="example-rendering-a-cube" class="level4">
<h4 class="anchored" data-anchor-id="example-rendering-a-cube">Example: Rendering a Cube</h4>
<ul>
<li>Start with cube vertices in local coordinates (<span class="math inline">\([-1,1]^3\)</span>).</li>
<li>Apply a scaling matrix to stretch it.</li>
<li>Apply a rotation matrix to tilt it.</li>
<li>Apply a translation matrix to move it into the scene.</li>
<li>Apply a projection matrix to flatten it onto the screen.</li>
</ul>
<p>Every step is linear algebra, and the final picture is the result of multiplying many matrices in sequence.</p>
</section>
<section id="robotics-connection" class="level4">
<h4 class="anchored" data-anchor-id="robotics-connection">Robotics Connection</h4>
<p>Robotic arms use similar pipelines: each joint contributes a rotation or translation, encoded as a matrix. By multiplying them, we get the forward kinematics-the position and orientation of the hand given the joint angles.</p>
</section>
<section id="why-it-matters-87" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-87">Why It Matters</h4>
<p>Geometry pipelines unify graphics, robotics, and vision. They show how linear algebra powers the everyday visuals of video games, animations, simulations, and even self-driving cars. Without the consistency of matrix multiplication, the complexity of managing transformations would be unmanageable.</p>
</section>
<section id="try-it-yourself-90" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-90">Try It Yourself</h4>
<ol type="1">
<li>Write down the sequence of matrices that rotate a square by 45°, scale it by 2, and translate it by <span class="math inline">\((3, 1)\)</span>. Multiply them to get the combined transformation.</li>
<li>Construct a cube in 3D and simulate a perspective projection by hand for one vertex.</li>
<li>For a simple 2-joint robotic arm, represent each joint with a rotation matrix and compute the final hand position.</li>
<li>Prove that composing affine transformations is closed under multiplication-why does this make pipelines possible?</li>
</ol>
<p>Geometry pipelines are the bridge between abstract linear algebra and tangible visual and mechanical systems. They are how math becomes movement, light, and image.</p>
</section>
</section>
<section id="computer-graphics-and-robotics-homogeneous-tricks-in-action" class="level3">
<h3 class="anchored" data-anchor-id="computer-graphics-and-robotics-homogeneous-tricks-in-action">92. Computer Graphics and Robotics (Homogeneous Tricks in Action)</h3>
<p>Linear algebra doesn’t just stay on the chalkboard-it drives the engines of computer graphics and robotics. Both fields need to describe and manipulate objects in space, often moving between multiple coordinate systems. The homogeneous coordinate trick-adding one extra dimension-makes this elegant: translations, scalings, and rotations all fit into a single framework of matrix multiplication. This uniformity allows efficient computation and consistent pipelines.</p>
<section id="homogeneous-coordinates-recap" class="level4">
<h4 class="anchored" data-anchor-id="homogeneous-coordinates-recap">Homogeneous Coordinates Recap</h4>
<p>In 2D, a point <span class="math inline">\((x, y)\)</span> becomes <span class="math inline">\([x, y, 1]^T\)</span>. In 3D, a point <span class="math inline">\((x, y, z)\)</span> becomes <span class="math inline">\([x, y, z, 1]^T\)</span>.</p>
<p>Why add the extra 1? Because then translations-normally not linear-become linear in the higher-dimensional embedding. Every affine transformation (rotations, scalings, shears, reflections, and translations) is just a single multiplication by a homogeneous matrix.</p>
<p>Example:</p>
<p><span class="math display">\[
T = \begin{bmatrix}
1 &amp; 0 &amp; 0 &amp; t_x \\
0 &amp; 1 &amp; 0 &amp; t_y \\
0 &amp; 0 &amp; 1 &amp; t_z \\
0 &amp; 0 &amp; 0 &amp; 1
\end{bmatrix}, \quad
p_h' = T p_h.
\]</span></p>
<p>This trick makes pipelines modular: just multiply the matrices in order.</p>
</section>
<section id="computer-graphics-pipelines" class="level4">
<h4 class="anchored" data-anchor-id="computer-graphics-pipelines">Computer Graphics Pipelines</h4>
<p>Graphics engines (like OpenGL or DirectX) rely entirely on homogeneous transformations:</p>
<ol type="1">
<li><p>Model Matrix: Puts the object in the scene.</p>
<ul>
<li>Example: Rotate a car 90° and translate it 10 units forward.</li>
</ul></li>
<li><p>View Matrix: Positions the virtual camera.</p>
<ul>
<li>Equivalent to moving the world so the camera sits at the origin.</li>
</ul></li>
<li><p>Projection Matrix: Projects 3D points to 2D.</p>
<ul>
<li>Perspective projection shrinks faraway objects, orthographic doesn’t.</li>
</ul></li>
<li><p>Viewport Matrix: Converts normalized 2D coordinates into screen pixels.</p></li>
</ol>
<p>Every pixel you see in a video game has passed through this stack of matrices.</p>
</section>
<section id="robotics-pipelines" class="level4">
<h4 class="anchored" data-anchor-id="robotics-pipelines">Robotics Pipelines</h4>
<p>In robotics, the same principle applies:</p>
<ul>
<li>A robot arm with joints is modeled as a chain of rigid-body transformations.</li>
<li>Each joint contributes a rotation or translation matrix.</li>
<li>Multiplying them gives the final pose of the robot’s end-effector (hand, tool, or gripper).</li>
</ul>
<p>This is called forward kinematics.</p>
<p>Example: A 2D robotic arm with two joints:</p>
<p><span class="math display">\[
p = R(\theta_1) T(l_1) R(\theta_2) T(l_2) \begin{bmatrix} 0 \\ 0 \\ 1 \end{bmatrix}.
\]</span></p>
<p>Here <span class="math inline">\(R(\theta_i)\)</span> are rotation matrices and <span class="math inline">\(T(l_i)\)</span> are translations along the arm length. Multiplying them gives the position of the hand.</p>
</section>
<section id="shared-challenges-in-graphics-and-robotics" class="level4">
<h4 class="anchored" data-anchor-id="shared-challenges-in-graphics-and-robotics">Shared Challenges in Graphics and Robotics</h4>
<ol type="1">
<li>Precision: Numerical round-off errors can accumulate; stable algorithms are critical.</li>
<li>Speed: Both fields demand real-time computation-60 frames per second for graphics, millisecond reaction times for robots.</li>
<li>Hierarchy: Objects in graphics may be nested (a car’s wheel rotates relative to the car), just like robot joints. Homogeneous transforms naturally handle these hierarchies.</li>
<li>Inverse Problems: Graphics uses inverse transforms for camera movement; robotics uses them for inverse kinematics (finding joint angles to reach a point).</li>
</ol>
</section>
<section id="why-homogeneous-tricks-are-powerful" class="level4">
<h4 class="anchored" data-anchor-id="why-homogeneous-tricks-are-powerful">Why Homogeneous Tricks Are Powerful</h4>
<ul>
<li>Uniformity: One system (matrix multiplication) handles all transformations.</li>
<li>Efficiency: Hardware (GPUs, controllers) can optimize matrix operations directly.</li>
<li>Scalability: Works the same in 2D, 3D, or higher.</li>
<li>Composability: Long pipelines are just products of matrices, avoiding special cases.</li>
</ul>
</section>
<section id="applications-49" class="level4">
<h4 class="anchored" data-anchor-id="applications-49">Applications</h4>
<ul>
<li>Graphics: Rendering engines, VR/AR, CAD software, motion capture.</li>
<li>Robotics: Arm manipulators, drones, autonomous vehicles, humanoid robots.</li>
<li>Crossover: Simulation platforms use the same math to test robots and render virtual environments.</li>
</ul>
</section>
<section id="try-it-yourself-91" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-91">Try It Yourself</h4>
<ol type="1">
<li>Build a 2D transformation pipeline: rotate a triangle, translate it, and project it into screen space. Write down the final transformation matrix.</li>
<li>Model a simple 2-joint robotic arm. Derive the forward kinematics using homogeneous matrices.</li>
<li>Implement a camera transform: place a cube at <span class="math inline">\((0,0,5)\)</span>, move the camera to <span class="math inline">\((0,0,0)\)</span>, and compute its 2D screen projection.</li>
<li>Show that composing a rotation and translation directly is equivalent to embedding them into a homogeneous matrix and multiplying.</li>
</ol>
<p>Homogeneous coordinates are the hidden secret that lets graphics and robots share the same mathematical DNA. They unify how we move pixels, machines, and virtual worlds.</p>
</section>
</section>
<section id="graphs-adjacency-and-laplacians-networks-via-matrices" class="level3">
<h3 class="anchored" data-anchor-id="graphs-adjacency-and-laplacians-networks-via-matrices">93. Graphs, Adjacency, and Laplacians (Networks via Matrices)</h3>
<p>Linear algebra provides a powerful language for studying graphs-networks of nodes connected by edges. From social networks to electrical circuits, from the internet’s structure to biological pathways, graphs appear everywhere. Matrices give graphs a numerical form, making it possible to analyze their structure using algebraic techniques.</p>
<section id="graph-basics-recap" class="level4">
<h4 class="anchored" data-anchor-id="graph-basics-recap">Graph Basics Recap</h4>
<ul>
<li>A graph <span class="math inline">\(G = (V, E)\)</span> has a set of vertices <span class="math inline">\(V\)</span> (nodes) and edges <span class="math inline">\(E\)</span> (connections).</li>
<li>Graphs may be undirected or directed, weighted or unweighted.</li>
<li>Many graph properties-connectivity, flow, clusters-can be studied through matrices.</li>
</ul>
</section>
<section id="the-adjacency-matrix" class="level4">
<h4 class="anchored" data-anchor-id="the-adjacency-matrix">The Adjacency Matrix</h4>
<p>For a graph with <span class="math inline">\(n\)</span> vertices, the adjacency matrix <span class="math inline">\(A \in \mathbb{R}^{n \times n}\)</span> encodes connections:</p>
<p><span class="math display">\[
A_{ij} = \begin{cases}
w_{ij}, &amp; \text{if there is an edge from node \(i\) to node \(j\)} \\
0, &amp; \text{otherwise}
\end{cases}
\]</span></p>
<ul>
<li>Unweighted graphs: entries are 0 or 1.</li>
<li>Weighted graphs: entries are edge weights (distances, costs, capacities).</li>
<li>Undirected graphs: <span class="math inline">\(A\)</span> is symmetric.</li>
<li>Directed graphs: <span class="math inline">\(A\)</span> may be asymmetric.</li>
</ul>
<p>The adjacency matrix is the algebraic fingerprint of the graph.</p>
</section>
<section id="powers-of-the-adjacency-matrix" class="level4">
<h4 class="anchored" data-anchor-id="powers-of-the-adjacency-matrix">Powers of the Adjacency Matrix</h4>
<p>The entry <span class="math inline">\((A^k)_{ij}\)</span> counts the number of walks of length <span class="math inline">\(k\)</span> from node <span class="math inline">\(i\)</span> to node <span class="math inline">\(j\)</span>.</p>
<ul>
<li><span class="math inline">\(A^2\)</span> tells how many two-step connections exist.</li>
<li>This property is used in algorithms for detecting paths, clustering, and network flow.</li>
</ul>
</section>
<section id="the-degree-matrix" class="level4">
<h4 class="anchored" data-anchor-id="the-degree-matrix">The Degree Matrix</h4>
<p>The degree of a vertex is the number of edges connected to it (or the sum of weights in weighted graphs).</p>
<p>The degree matrix <span class="math inline">\(D\)</span> is diagonal:</p>
<p><span class="math display">\[
D_{ii} = \sum_j A_{ij}.
\]</span></p>
<p>This matrix measures how “connected” each node is.</p>
</section>
<section id="the-graph-laplacian" class="level4">
<h4 class="anchored" data-anchor-id="the-graph-laplacian">The Graph Laplacian</h4>
<p>The combinatorial Laplacian is defined as:</p>
<p><span class="math display">\[
L = D - A.
\]</span></p>
<p>Key properties:</p>
<ul>
<li><span class="math inline">\(L\)</span> is symmetric (for undirected graphs).</li>
<li>Each row sums to zero.</li>
<li>The smallest eigenvalue is always 0, with eigenvector <span class="math inline">\([1, 1, \dots, 1]^T\)</span>.</li>
</ul>
<p>The Laplacian encodes connectivity: if the graph splits into <span class="math inline">\(k\)</span> connected components, then <span class="math inline">\(L\)</span> has exactly <span class="math inline">\(k\)</span> zero eigenvalues.</p>
</section>
<section id="normalized-laplacians" class="level4">
<h4 class="anchored" data-anchor-id="normalized-laplacians">Normalized Laplacians</h4>
<p>Two common normalized versions are:</p>
<p><span class="math display">\[
L_{sym} = D^{-1/2} L D^{-1/2}, \quad L_{rw} = D^{-1} L.
\]</span></p>
<p>These rescale the Laplacian for applications like spectral clustering.</p>
</section>
<section id="spectral-graph-theory" class="level4">
<h4 class="anchored" data-anchor-id="spectral-graph-theory">Spectral Graph Theory</h4>
<p>Eigenvalues and eigenvectors of <span class="math inline">\(A\)</span> or <span class="math inline">\(L\)</span> reveal structure:</p>
<ul>
<li>Algebraic connectivity: The second-smallest eigenvalue of <span class="math inline">\(L\)</span> measures how well connected the graph is.</li>
<li>Spectral clustering: Eigenvectors of <span class="math inline">\(L\)</span> partition graphs into communities.</li>
<li>Random walks: Transition probabilities relate to <span class="math inline">\(D^{-1}A\)</span>.</li>
</ul>
</section>
<section id="example-a-simple-graph" class="level4">
<h4 class="anchored" data-anchor-id="example-a-simple-graph">Example: A Simple Graph</h4>
<p>Take a triangle graph with 3 nodes, each connected to the other two.</p>
<p><span class="math display">\[
A = \begin{bmatrix}
0 &amp; 1 &amp; 1 \\
1 &amp; 0 &amp; 1 \\
1 &amp; 1 &amp; 0
\end{bmatrix}, \quad
D = \begin{bmatrix}
2 &amp; 0 &amp; 0 \\
0 &amp; 2 &amp; 0 \\
0 &amp; 0 &amp; 2
\end{bmatrix}, \quad
L = \begin{bmatrix}
2 &amp; -1 &amp; -1 \\
-1 &amp; 2 &amp; -1 \\
-1 &amp; -1 &amp; 2
\end{bmatrix}.
\]</span></p>
<ul>
<li>Eigenvalues of <span class="math inline">\(L\)</span>: <span class="math inline">\(0, 3, 3\)</span>.</li>
<li>The single zero eigenvalue confirms the graph is connected.</li>
</ul>
</section>
<section id="applications-50" class="level4">
<h4 class="anchored" data-anchor-id="applications-50">Applications</h4>
<ol type="1">
<li>Community Detection: Spectral clustering finds natural divisions in social or biological networks.</li>
<li>Graph Drawing: Eigenvectors of <span class="math inline">\(L\)</span> provide coordinates for visually embedding graphs.</li>
<li>Random Walks &amp; PageRank: Transition matrices from adjacency define importance scores.</li>
<li>Physics: Laplacians appear in discrete versions of diffusion and vibration problems.</li>
<li>Machine Learning: Graph neural networks (GNNs) use Laplacians to propagate signals across graph structure.</li>
</ol>
</section>
<section id="why-it-matters-88" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-88">Why It Matters</h4>
<p>Graphs and matrices are two sides of the same coin: one combinatorial, one algebraic. By turning a network into a matrix, linear algebra gives us access to the full toolbox of eigenvalues, norms, and factorizations, enabling deep insights into connectivity, flow, and structure.</p>
</section>
<section id="try-it-yourself-92" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-92">Try It Yourself</h4>
<ol type="1">
<li>Compute adjacency, degree, and Laplacian matrices for a square graph (4 nodes in a cycle). Find eigenvalues of <span class="math inline">\(L\)</span>.</li>
<li>Prove that the Laplacian always has at least one zero eigenvalue.</li>
<li>Show that if a graph has <span class="math inline">\(k\)</span> components, then the multiplicity of zero as an eigenvalue is exactly <span class="math inline">\(k\)</span>.</li>
<li>For a random walk on a graph, derive the transition matrix <span class="math inline">\(P = D^{-1}A\)</span>. Interpret its eigenvectors.</li>
</ol>
<p>Graphs demonstrate how linear algebra stretches beyond geometry and data tables-it becomes a universal language for networks, from molecules to megacities.</p>
</section>
</section>
<section id="data-preprocessing-as-linear-operations-centering-whitening-scaling" class="level3">
<h3 class="anchored" data-anchor-id="data-preprocessing-as-linear-operations-centering-whitening-scaling">94. Data Preprocessing as Linear Operations (Centering, Whitening, Scaling)</h3>
<p>Before any sophisticated model can be trained, raw data must be preprocessed. Surprisingly, many of the most common preprocessing steps-centering, scaling, whitening-are nothing more than linear algebra operations in disguise. Understanding them this way not only clarifies why they work, but also shows how they connect to broader concepts like covariance, eigenvalues, and singular value decomposition.</p>
<section id="the-nature-of-preprocessing" class="level4">
<h4 class="anchored" data-anchor-id="the-nature-of-preprocessing">The Nature of Preprocessing</h4>
<p>Most datasets are stored as a matrix: rows correspond to samples (observations) and columns correspond to features (variables). For instance, in a dataset of 1,000 people with height, weight, and age recorded, we’d have a <span class="math inline">\(1000 \times 3\)</span> matrix. Linear algebra allows us to systematically reshape, scale, and rotate this matrix to prepare it for downstream analysis.</p>
</section>
<section id="centering-shifting-the-origin" class="level4">
<h4 class="anchored" data-anchor-id="centering-shifting-the-origin">Centering: Shifting the Origin</h4>
<p>Centering means subtracting the mean of each column (feature) from all entries in that column.</p>
<p><span class="math display">\[
X_{centered} = X - \mathbf{1}\mu^T
\]</span></p>
<ul>
<li>Here <span class="math inline">\(X\)</span> is the data matrix, <span class="math inline">\(\mu\)</span> is the vector of column means, and <span class="math inline">\(\mathbf{1}\)</span> is a column of ones.</li>
<li>Effect: moves the dataset so that each feature has mean zero.</li>
<li>Geometric view: translates the cloud of points so its “center of mass” sits at the origin.</li>
<li>Why important: covariance and correlation formulas assume data are mean-centered; otherwise, cross-terms are skewed.</li>
</ul>
<p>Example: If people’s heights average 170 cm, subtract 170 from every height. After centering, “height = 0” corresponds to the average person.</p>
</section>
<section id="scaling-normalizing-variability" class="level4">
<h4 class="anchored" data-anchor-id="scaling-normalizing-variability">Scaling: Normalizing Variability</h4>
<p>Raw features can have different units or magnitudes (e.g., weight in kg, income in thousands of dollars). To compare them fairly, we scale:</p>
<p><span class="math display">\[
X_{scaled} = X D^{-1}
\]</span></p>
<p>where <span class="math inline">\(D\)</span> is a diagonal matrix of feature standard deviations.</p>
<ul>
<li>Each feature now has variance 1.</li>
<li>Geometric view: rescales axes so all dimensions have equal “spread.”</li>
<li>Common in machine learning: ensures gradient descent does not disproportionately focus on features with large raw values.</li>
</ul>
<p>Example: If weight varies around 60 kg ± 15, dividing by 15 makes its spread comparable to that of height (±10 cm).</p>
</section>
<section id="whitening-removing-correlations" class="level4">
<h4 class="anchored" data-anchor-id="whitening-removing-correlations">Whitening: Removing Correlations</h4>
<p>Even after centering and scaling, features can remain correlated (e.g., height and weight). Whitening transforms the data so features become uncorrelated with unit variance.</p>
<ul>
<li>Let <span class="math inline">\(\Sigma = \frac{1}{n} X^T X\)</span> be the covariance matrix of centered data.</li>
<li>Perform eigendecomposition: <span class="math inline">\(\Sigma = Q \Lambda Q^T\)</span>.</li>
<li>Whitening transform:</li>
</ul>
<p><span class="math display">\[
X_{white} = X Q \Lambda^{-1/2} Q^T
\]</span></p>
<p>Result:</p>
<ol type="1">
<li>The covariance matrix of <span class="math inline">\(X_{white}\)</span> is the identity matrix.</li>
<li>Each new feature is a rotated combination of old features, with no redundancy.</li>
</ol>
<p>Geometric view: whitening “spheres” the data cloud, turning an ellipse into a perfect circle.</p>
</section>
<section id="covariance-matrix-as-the-key-player" class="level4">
<h4 class="anchored" data-anchor-id="covariance-matrix-as-the-key-player">Covariance Matrix as the Key Player</h4>
<p>The covariance matrix itself arises naturally from preprocessing:</p>
<p><span class="math display">\[
\Sigma = \frac{1}{n} X^T X \quad \text{(if \(X\) is centered).}
\]</span></p>
<ul>
<li>Diagonal entries: variances of features.</li>
<li>Off-diagonal entries: covariances, measuring linear relationships.</li>
<li>Preprocessing operations (centering, scaling, whitening) are designed to reshape data so <span class="math inline">\(\Sigma\)</span> becomes easier to interpret and more stable for learning algorithms.</li>
</ul>
</section>
<section id="connections-to-pca" class="level4">
<h4 class="anchored" data-anchor-id="connections-to-pca">Connections to PCA</h4>
<ul>
<li>Centering is required before PCA, otherwise the first component just points to the mean.</li>
<li>Scaling ensures PCA does not overweight large-variance features.</li>
<li>Whitening is closely related to PCA itself: PCA diagonalizes the covariance, and whitening goes one step further by rescaling eigenvalues to unity.</li>
</ul>
<p>Thus, PCA can be seen as a preprocessing pipeline plus an analysis step.</p>
</section>
<section id="practical-workflows" class="level4">
<h4 class="anchored" data-anchor-id="practical-workflows">Practical Workflows</h4>
<ol type="1">
<li>Centering and Scaling (Standardization): The default for many algorithms like logistic regression or SVM.</li>
<li>Whitening: Often used in signal processing (e.g., removing correlations in audio or images).</li>
<li>Batch Normalization in Deep Learning: A variant of centering + scaling applied layer by layer during training.</li>
<li>Whitening in Image Processing: Ensures features like pixel intensities are decorrelated, improving compression and recognition.</li>
</ol>
</section>
<section id="worked-example" class="level4">
<h4 class="anchored" data-anchor-id="worked-example">Worked Example</h4>
<p>Suppose we have three features: height, weight, and age.</p>
<ol type="1">
<li><p>Raw data:</p>
<ul>
<li>Mean height = 170 cm, mean weight = 65 kg, mean age = 35 years.</li>
<li>Variance differs widely: age varies less, weight more.</li>
</ul></li>
<li><p>After centering:</p>
<ul>
<li>Mean of each feature is zero.</li>
<li>A person of average height now has value 0 in that feature.</li>
</ul></li>
<li><p>After scaling:</p>
<ul>
<li>All features have unit variance.</li>
<li>Algorithms can treat age and weight equally.</li>
</ul></li>
<li><p>After whitening:</p>
<ul>
<li>Correlation between height and weight disappears.</li>
<li>Features become orthogonal directions in feature space.</li>
</ul></li>
</ol>
</section>
<section id="why-it-matters-89" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-89">Why It Matters</h4>
<p>Without preprocessing, models may be misled by scale, units, or correlations. Preprocessing makes features comparable, balanced, and independent-a crucial condition for algorithms that rely on geometry (distances, angles, inner products).</p>
<p>In essence, preprocessing is the bridge from messy, real-world data to the clean structures linear algebra expects.</p>
</section>
<section id="try-it-yourself-93" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-93">Try It Yourself</h4>
<ol type="1">
<li>For a small dataset, compute the covariance matrix before and after centering. What changes?</li>
<li>Scale the dataset so each feature has unit variance. Check the new covariance.</li>
<li>Perform whitening via eigendecomposition and verify the covariance matrix becomes the identity.</li>
<li>Plot the data points in 2D before and after whitening. Notice how an ellipse becomes a circle.</li>
</ol>
<p>Preprocessing through linear algebra shows that preparing data is not just housekeeping-it’s a fundamental reshaping of the problem’s geometry.</p>
</section>
</section>
<section id="linear-regression-and-classification-from-model-to-matrix" class="level3">
<h3 class="anchored" data-anchor-id="linear-regression-and-classification-from-model-to-matrix">95. Linear Regression and Classification (From Model to Matrix)</h3>
<p>Linear algebra provides the foundation for two of the most widely used tools in data science and applied statistics: linear regression (predicting continuous outcomes) and linear classification (separating categories). Both problems reduce to expressing data in matrix form and then applying linear operations to estimate parameters.</p>
<section id="the-regression-setup" class="level4">
<h4 class="anchored" data-anchor-id="the-regression-setup">The Regression Setup</h4>
<p>Suppose we want to predict an output <span class="math inline">\(y \in \mathbb{R}^n\)</span> from features collected in a data matrix <span class="math inline">\(X \in \mathbb{R}^{n \times p}\)</span>, where:</p>
<ul>
<li><span class="math inline">\(n\)</span> = number of observations (samples).</li>
<li><span class="math inline">\(p\)</span> = number of features (variables).</li>
</ul>
<p>We assume a linear model:</p>
<p><span class="math display">\[
y \approx X\beta,
\]</span></p>
<p>where <span class="math inline">\(\beta \in \mathbb{R}^p\)</span> is the vector of coefficients (weights). Each entry of <span class="math inline">\(\beta\)</span> tells us how much its feature contributes to the prediction.</p>
</section>
<section id="the-normal-equations" class="level4">
<h4 class="anchored" data-anchor-id="the-normal-equations">The Normal Equations</h4>
<p>We want to minimize the squared error:</p>
<p><span class="math display">\[
\min_\beta \|y - X\beta\|^2.
\]</span></p>
<p>Differentiating leads to the normal equations:</p>
<p><span class="math display">\[
X^T X \beta = X^T y.
\]</span></p>
<ul>
<li>If <span class="math inline">\(X^T X\)</span> is invertible:</li>
</ul>
<p><span class="math display">\[
\hat{\beta} = (X^T X)^{-1} X^T y.
\]</span></p>
<ul>
<li>If not invertible (multicollinearity, too many features), we use the pseudoinverse via SVD:</li>
</ul>
<p><span class="math display">\[
\hat{\beta} = X^+ y.
\]</span></p>
</section>
<section id="geometric-interpretation-18" class="level4">
<h4 class="anchored" data-anchor-id="geometric-interpretation-18">Geometric Interpretation</h4>
<ul>
<li><span class="math inline">\(X\beta\)</span> is the projection of <span class="math inline">\(y\)</span> onto the column space of <span class="math inline">\(X\)</span>.</li>
<li>The residual <span class="math inline">\(r = y - X\hat{\beta}\)</span> is orthogonal to all columns of <span class="math inline">\(X\)</span>.</li>
<li>This “closest fit” property is why regression is a projection problem.</li>
</ul>
</section>
<section id="classification-with-linear-models" class="level4">
<h4 class="anchored" data-anchor-id="classification-with-linear-models">Classification with Linear Models</h4>
<p>Instead of predicting continuous outputs, sometimes we want to separate categories (e.g., spam vs.&nbsp;not spam).</p>
<ul>
<li>Linear classifier: decides based on the sign of a linear function.</li>
</ul>
<p><span class="math display">\[
\hat{y} = \text{sign}(w^T x + b).
\]</span></p>
<ul>
<li>Geometric view: <span class="math inline">\(w\)</span> defines a hyperplane in feature space. Points on one side are labeled positive, on the other side negative.</li>
<li>Relation to regression: logistic regression replaces squared error with a log-likelihood loss, but still solves for weights via iterative linear-algebraic methods.</li>
</ul>
</section>
<section id="multiclass-extension" class="level4">
<h4 class="anchored" data-anchor-id="multiclass-extension">Multiclass Extension</h4>
<ul>
<li>For <span class="math inline">\(k\)</span> classes, we use a weight matrix <span class="math inline">\(W \in \mathbb{R}^{p \times k}\)</span>.</li>
<li>Prediction:</li>
</ul>
<p><span class="math display">\[
\hat{y} = \arg \max_j (XW)_{ij}.
\]</span></p>
<ul>
<li>Each class has a column of <span class="math inline">\(W\)</span>, and the classifier picks the column with the largest score.</li>
</ul>
</section>
<section id="example-predicting-house-prices" class="level4">
<h4 class="anchored" data-anchor-id="example-predicting-house-prices">Example: Predicting House Prices</h4>
<ul>
<li>Features: size, number of rooms, distance to city center.</li>
<li>Target: price.</li>
<li><span class="math inline">\(X\)</span> = matrix of features, <span class="math inline">\(y\)</span> = price vector.</li>
<li>Regression solves for coefficients showing how strongly each factor influences price.</li>
</ul>
<p>If we switch to classification (predicting “expensive” vs.&nbsp;“cheap”), we treat price as a label and solve for a hyperplane separating the two categories.</p>
</section>
<section id="computational-aspects" class="level4">
<h4 class="anchored" data-anchor-id="computational-aspects">Computational Aspects</h4>
<ul>
<li>Directly solving normal equations: <span class="math inline">\(O(p^3)\)</span> (matrix inversion).</li>
<li>QR factorization: numerically more stable.</li>
<li>SVD: best when <span class="math inline">\(X\)</span> is ill-conditioned or rank-deficient.</li>
<li>Modern libraries: exploit sparsity or use gradient-based methods for large datasets.</li>
</ul>
</section>
<section id="connections-to-other-topics" class="level4">
<h4 class="anchored" data-anchor-id="connections-to-other-topics">Connections to Other Topics</h4>
<ul>
<li>Least Squares (Chapter 8): Regression is the canonical least-squares problem.</li>
<li>SVD (Chapter 9): Pseudoinverse gives regression when columns are dependent.</li>
<li>Regularization (Chapter 9): Ridge regression adds a penalty <span class="math inline">\(\lambda \|\beta\|^2\)</span> to improve stability.</li>
<li>Classification (Chapter 10): Forms the foundation of more complex models like support vector machines and neural networks.</li>
</ul>
</section>
<section id="why-it-matters-90" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-90">Why It Matters</h4>
<p>Linear regression and classification show the direct link between linear algebra and real-world decisions. They combine geometry (projection, hyperplanes), algebra (solving systems), and computation (factorizations). Despite their simplicity, they remain indispensable: they are interpretable, fast, and often competitive with more complex models.</p>
</section>
<section id="try-it-yourself-94" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-94">Try It Yourself</h4>
<ol type="1">
<li>Given three features and five samples, construct <span class="math inline">\(X\)</span> and <span class="math inline">\(y\)</span>. Solve for <span class="math inline">\(\beta\)</span> using the normal equations.</li>
<li>Show that residuals are orthogonal to all columns of <span class="math inline">\(X\)</span>.</li>
<li>Write down a linear classifier separating two clusters of points in 2D. Sketch the separating hyperplane.</li>
<li>Explore what happens when two features are highly correlated (collinear). Use pseudoinverse to recover a stable solution.</li>
</ol>
<p>Linear regression and classification are proof that linear algebra is not just abstract-it is the engine of practical prediction.</p>
</section>
</section>
<section id="pca-in-practice-dimensionality-reduction-workflow" class="level3">
<h3 class="anchored" data-anchor-id="pca-in-practice-dimensionality-reduction-workflow">96. PCA in Practice (Dimensionality Reduction Workflow)</h3>
<p>Principal Component Analysis (PCA) is one of the most widely used tools in applied linear algebra. At its heart, PCA identifies the directions (principal components) along which data varies the most, and then re-expresses the data in terms of those directions. In practice, PCA is not just a mathematical curiosity-it is a complete workflow for reducing dimensionality, denoising data, and extracting patterns from high-dimensional datasets.</p>
<section id="the-motivation" class="level4">
<h4 class="anchored" data-anchor-id="the-motivation">The Motivation</h4>
<p>Modern datasets often have thousands or even millions of features:</p>
<ul>
<li>Images: each pixel is a feature.</li>
<li>Genomics: each gene expression level is a feature.</li>
<li>Text: each word in a vocabulary becomes a dimension.</li>
</ul>
<p>Working in such high dimensions is expensive (computationally) and fragile (noise accumulates). PCA provides a systematic way to reduce the feature space to a smaller set of dimensions that still captures most of the variability.</p>
</section>
<section id="step-1-organizing-the-data" class="level4">
<h4 class="anchored" data-anchor-id="step-1-organizing-the-data">Step 1: Organizing the Data</h4>
<p>We start with a data matrix <span class="math inline">\(X \in \mathbb{R}^{n \times p}\)</span>:</p>
<ul>
<li><span class="math inline">\(n\)</span>: number of samples (observations).</li>
<li><span class="math inline">\(p\)</span>: number of features (variables).</li>
</ul>
<p>Each row is a sample; each column is a feature.</p>
<p>Centering is the first preprocessing step: subtract the mean of each column so the dataset has mean zero. This ensures that PCA describes variance rather than being biased by offsets.</p>
<p><span class="math display">\[
X_{centered} = X - \mathbf{1}\mu^T
\]</span></p>
</section>
<section id="step-2-covariance-matrix" class="level4">
<h4 class="anchored" data-anchor-id="step-2-covariance-matrix">Step 2: Covariance Matrix</h4>
<p>Next, compute the covariance matrix:</p>
<p><span class="math display">\[
\Sigma = \frac{1}{n} X_{centered}^T X_{centered}.
\]</span></p>
<ul>
<li>Diagonal entries: variance of each feature.</li>
<li>Off-diagonal entries: how features co-vary.</li>
</ul>
<p>The structure of <span class="math inline">\(\Sigma\)</span> determines the directions of maximal variation in the data.</p>
</section>
<section id="step-3-eigen-decomposition-or-svd" class="level4">
<h4 class="anchored" data-anchor-id="step-3-eigen-decomposition-or-svd">Step 3: Eigen-Decomposition or SVD</h4>
<p>Two equivalent approaches:</p>
<ol type="1">
<li><p>Eigen-decomposition: Solve <span class="math inline">\(\Sigma v = \lambda v\)</span>.</p>
<ul>
<li>Eigenvectors <span class="math inline">\(v\)</span> are the principal components.</li>
<li>Eigenvalues <span class="math inline">\(\lambda\)</span> measure variance along those directions.</li>
</ul></li>
<li><p>Singular Value Decomposition (SVD): Directly decompose the centered data matrix:</p>
<p><span class="math display">\[
X_{centered} = U \Sigma V^T.
\]</span></p>
<ul>
<li>Columns of <span class="math inline">\(V\)</span> = principal directions.</li>
<li>Squared singular values correspond to variances.</li>
</ul></li>
</ol>
<p>SVD is preferred in practice for numerical stability and efficiency, especially when <span class="math inline">\(p\)</span> is very large.</p>
</section>
<section id="step-4-choosing-the-number-of-components" class="level4">
<h4 class="anchored" data-anchor-id="step-4-choosing-the-number-of-components">Step 4: Choosing the Number of Components</h4>
<p>We order eigenvalues <span class="math inline">\(\lambda_1 \geq \lambda_2 \geq \dots \geq \lambda_p\)</span>.</p>
<ul>
<li><p>Explained variance ratio:</p>
<p><span class="math display">\[
\text{EVR}(k) = \frac{\sum_{i=1}^k \lambda_i}{\sum_{i=1}^p \lambda_i}.
\]</span></p></li>
<li><p>We choose <span class="math inline">\(k\)</span> such that EVR exceeds some threshold (e.g., 90–95%).</p></li>
<li><p>This balances dimensionality reduction with information preservation.</p></li>
</ul>
<p>Graphically, a scree plot shows eigenvalues, and we look for the “elbow” point where additional components add little variance.</p>
</section>
<section id="step-5-projecting-data" class="level4">
<h4 class="anchored" data-anchor-id="step-5-projecting-data">Step 5: Projecting Data</h4>
<p>Once we select <span class="math inline">\(k\)</span> components, we project onto them:</p>
<p><span class="math display">\[
X_{PCA} = X_{centered} V_k,
\]</span></p>
<p>where <span class="math inline">\(V_k\)</span> contains the top <span class="math inline">\(k\)</span> eigenvectors.</p>
<p>Result:</p>
<ul>
<li><span class="math inline">\(X_{PCA} \in \mathbb{R}^{n \times k}\)</span>.</li>
<li>Each row is now a <span class="math inline">\(k\)</span>-dimensional representation of the original sample.</li>
</ul>
</section>
<section id="worked-example-face-images" class="level4">
<h4 class="anchored" data-anchor-id="worked-example-face-images">Worked Example: Face Images</h4>
<p>Suppose we have a dataset of grayscale images, each <span class="math inline">\(100 \times 100\)</span> pixels (<span class="math inline">\(p = 10,000\)</span>).</p>
<ol type="1">
<li>Center each pixel value.</li>
<li>Compute covariance across all images.</li>
<li>Find eigenvectors = eigenfaces. These are characteristic patterns like “glasses,” “mouth shape,” or “lighting direction.”</li>
<li>Keep top 50 components. Each face is now represented as a 50-dimensional vector instead of 10,000.</li>
</ol>
<p>This drastically reduces storage and speeds up recognition while keeping key features.</p>
</section>
<section id="practical-considerations" class="level4">
<h4 class="anchored" data-anchor-id="practical-considerations">Practical Considerations</h4>
<ul>
<li>Standardization: If features have different scales (e.g., age in years vs.&nbsp;income in thousands), we must scale them before PCA.</li>
<li>Computational shortcuts: For very large <span class="math inline">\(p\)</span>, it’s often faster to compute PCA via truncated SVD on <span class="math inline">\(X\)</span> directly.</li>
<li>Noise filtering: Small eigenvalues often correspond to noise; truncating them denoises the dataset.</li>
<li>Interpretability: Principal components are linear combinations of features. Sometimes these combinations are interpretable, sometimes not.</li>
</ul>
</section>
<section id="connections-to-other-concepts" class="level4">
<h4 class="anchored" data-anchor-id="connections-to-other-concepts">Connections to Other Concepts</h4>
<ul>
<li>Whitening (Chapter 94): PCA followed by scaling eigenvalues to 1 is whitening.</li>
<li>SVD (Chapter 9): PCA is essentially an application of SVD.</li>
<li>Regression (Chapter 95): PCA can be used before regression to reduce collinearity among predictors (PCA regression).</li>
<li>Machine learning pipelines: PCA is often used before clustering, classification, or neural networks.</li>
</ul>
</section>
<section id="why-it-matters-91" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-91">Why It Matters</h4>
<p>PCA turns raw, unwieldy data into a compact form without losing essential structure. It enables visualization (2D/3D plots of high-dimensional data), faster learning, and noise reduction. Many breakthroughs-from face recognition to gene expression analysis-rely on PCA as the first preprocessing step.</p>
</section>
<section id="try-it-yourself-95" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-95">Try It Yourself</h4>
<ol type="1">
<li>Take a dataset with 3 features. Manually compute covariance, eigenvalues, and eigenvectors.</li>
<li>Project the data onto the first two principal components and plot. Compare to the original 3D scatter.</li>
<li>Download an image dataset and apply PCA to compress it. Reconstruct the images with 10, 50, 100 components. Observe the trade-off between compression and fidelity.</li>
<li>Compute explained variance ratios and decide how many components to keep.</li>
</ol>
<p>PCA is the bridge between raw data and meaningful representation: it reduces complexity while sharpening patterns. It shows how linear algebra can reveal hidden order in high-dimensional chaos.</p>
</section>
</section>
<section id="recommender-systems-and-low-rank-models-fill-the-missing-entries" class="level3">
<h3 class="anchored" data-anchor-id="recommender-systems-and-low-rank-models-fill-the-missing-entries">97. Recommender Systems and Low-Rank Models (Fill the Missing Entries)</h3>
<p>Recommender systems-such as those used by Netflix, Amazon, or Spotify-are built on the principle that preferences can be captured by low-dimensional structures hidden inside large, sparse data. Linear algebra gives us the machinery to expose and exploit these structures, especially through low-rank models.</p>
<section id="the-matrix-of-preferences" class="level4">
<h4 class="anchored" data-anchor-id="the-matrix-of-preferences">The Matrix of Preferences</h4>
<p>We begin with a user–item matrix <span class="math inline">\(R \in \mathbb{R}^{m \times n}\)</span>:</p>
<ul>
<li>Rows represent users.</li>
<li>Columns represent items (movies, books, songs).</li>
<li>Entries <span class="math inline">\(R_{ij}\)</span> store the rating (say 1–5 stars) or interaction (clicks, purchases).</li>
</ul>
<p>In practice, most entries are missing-users rate only a small subset of items. The central challenge: predict the missing entries.</p>
</section>
<section id="why-low-rank-structure" class="level4">
<h4 class="anchored" data-anchor-id="why-low-rank-structure">Why Low-Rank Structure?</h4>
<p>Despite its size, <span class="math inline">\(R\)</span> often lies close to a low-rank approximation:</p>
<p><span class="math display">\[
R \approx U V^T
\]</span></p>
<ul>
<li><span class="math inline">\(U \in \mathbb{R}^{m \times k}\)</span>: user factors.</li>
<li><span class="math inline">\(V \in \mathbb{R}^{n \times k}\)</span>: item factors.</li>
<li><span class="math inline">\(k \ll \min(m, n)\)</span>.</li>
</ul>
<p>Here, each user and each item is represented in a shared latent feature space.</p>
<ul>
<li>Example: For movies, latent dimensions might capture “action vs.&nbsp;romance,” “old vs.&nbsp;new,” or “mainstream vs.&nbsp;indie.”</li>
<li>A user’s preference vector in this space interacts with an item’s feature vector to generate a predicted rating.</li>
</ul>
<p>This factorization explains correlations: if you liked Movie A and B, and Movie C shares similar latent features, the system predicts you’ll like C too.</p>
</section>
<section id="singular-value-decomposition-svd-approach" class="level4">
<h4 class="anchored" data-anchor-id="singular-value-decomposition-svd-approach">Singular Value Decomposition (SVD) Approach</h4>
<p>If <span class="math inline">\(R\)</span> were complete (no missing entries), we could compute the SVD:</p>
<p><span class="math display">\[
R = U \Sigma V^T.
\]</span></p>
<ul>
<li>Keep the top <span class="math inline">\(k\)</span> singular values to form a rank-<span class="math inline">\(k\)</span> approximation.</li>
<li>This captures the dominant patterns in user preferences.</li>
<li>Geometric view: project the massive data cloud onto a smaller <span class="math inline">\(k\)</span>-dimensional subspace where structure is clearer.</li>
</ul>
<p>But real data is incomplete. That leads to matrix completion problems.</p>
</section>
<section id="matrix-completion" class="level4">
<h4 class="anchored" data-anchor-id="matrix-completion">Matrix Completion</h4>
<p>Matrix completion tries to infer missing entries of <span class="math inline">\(R\)</span> by assuming low rank. The optimization problem is:</p>
<p><span class="math display">\[
\min_{X} \ \text{rank}(X) \quad \text{s.t. } X_{ij} = R_{ij} \text{ for observed entries}.
\]</span></p>
<p>Since minimizing rank is NP-hard, practical algorithms instead minimize the nuclear norm (sum of singular values) or use alternating minimization:</p>
<ul>
<li>Initialize <span class="math inline">\(U, V\)</span> randomly.</li>
<li>Iteratively solve for one while fixing the other.</li>
<li>Converge to a low-rank factorization that fits the observed ratings.</li>
</ul>
</section>
<section id="alternating-least-squares-als" class="level4">
<h4 class="anchored" data-anchor-id="alternating-least-squares-als">Alternating Least Squares (ALS)</h4>
<p>ALS is a standard approach:</p>
<ol type="1">
<li>Fix <span class="math inline">\(V\)</span>, solve least squares for <span class="math inline">\(U\)</span>.</li>
<li>Fix <span class="math inline">\(U\)</span>, solve least squares for <span class="math inline">\(V\)</span>.</li>
<li>Repeat until convergence.</li>
</ol>
<p>Each subproblem is straightforward linear regression, solvable with normal equations or QR decomposition.</p>
</section>
<section id="stochastic-gradient-descent-sgd" class="level4">
<h4 class="anchored" data-anchor-id="stochastic-gradient-descent-sgd">Stochastic Gradient Descent (SGD)</h4>
<p>Another approach: treat each observed rating as a training sample. Update latent vectors by minimizing squared error:</p>
<p><span class="math display">\[
\ell = (R_{ij} - u_i^T v_j)^2.
\]</span></p>
<p>Iteratively adjust user vector <span class="math inline">\(u_i\)</span> and item vector <span class="math inline">\(v_j\)</span> along gradients. This scales well to huge datasets, making it common in practice.</p>
</section>
<section id="regularization" class="level4">
<h4 class="anchored" data-anchor-id="regularization">Regularization</h4>
<p>To prevent overfitting:</p>
<p><span class="math display">\[
\ell = (R_{ij} - u_i^T v_j)^2 + \lambda (\|u_i\|^2 + \|v_j\|^2).
\]</span></p>
<ul>
<li>Regularization shrinks factors, discouraging extreme values.</li>
<li>Geometrically, it keeps latent vectors within a reasonable ball in feature space.</li>
</ul>
</section>
<section id="cold-start-problem" class="level4">
<h4 class="anchored" data-anchor-id="cold-start-problem">Cold Start Problem</h4>
<ul>
<li>New users: Without ratings, <span class="math inline">\(u_i\)</span> is unknown. Solutions: use demographic features or ask for a few initial ratings.</li>
<li>New items: Similarly, items need side information (metadata, tags) to generate initial latent vectors.</li>
</ul>
<p>This is where hybrid models combine matrix factorization with content-based features.</p>
</section>
<section id="example-movie-ratings" class="level4">
<h4 class="anchored" data-anchor-id="example-movie-ratings">Example: Movie Ratings</h4>
<p>Imagine 1,000 users and 5,000 movies.</p>
<ul>
<li>The raw <span class="math inline">\(R\)</span> matrix has 5 million entries, but each user has rated only ~50 movies.</li>
<li>Matrix completion with rank <span class="math inline">\(k = 20\)</span> recovers a dense approximation.</li>
<li>Each user is represented by 20 latent “taste” factors; each movie by 20 latent “theme” factors.</li>
<li>Prediction: the dot product of user and movie vectors.</li>
</ul>
</section>
<section id="beyond-ratings-implicit-feedback" class="level4">
<h4 class="anchored" data-anchor-id="beyond-ratings-implicit-feedback">Beyond Ratings: Implicit Feedback</h4>
<p>In practice, systems often lack explicit ratings. Instead, they use:</p>
<ul>
<li>Views, clicks, purchases, skips.</li>
<li>These signals are indirect but abundant.</li>
<li>Factorization can handle them by treating interactions as weighted observations.</li>
</ul>
</section>
<section id="connections-to-other-linear-algebra-tools" class="level4">
<h4 class="anchored" data-anchor-id="connections-to-other-linear-algebra-tools">Connections to Other Linear Algebra Tools</h4>
<ul>
<li>SVD (Chapter 9): The backbone of factorization methods.</li>
<li>Pseudoinverse (Chapter 9): Useful when solving small regression subproblems in ALS.</li>
<li>Conditioning (Chapter 9): Factorization stability depends on well-scaled latent factors.</li>
<li>PCA (Chapter 96): PCA is essentially a low-rank approximation, so PCA and recommenders share the same mathematics.</li>
</ul>
</section>
<section id="why-it-matters-92" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-92">Why It Matters</h4>
<p>Recommender systems personalize the modern internet. Every playlist suggestion, book recommendation, or ad placement is powered by linear algebra hidden in a massive sparse matrix. Low-rank modeling shows how even incomplete, noisy data can be harnessed to reveal patterns of preference and behavior.</p>
</section>
<section id="try-it-yourself-96" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-96">Try It Yourself</h4>
<ol type="1">
<li>Take a small user–item matrix with missing entries. Apply rank-2 approximation via SVD to fill in gaps.</li>
<li>Implement one step of ALS: fix movie factors and update user factors with least squares.</li>
<li>Compare predictions with and without regularization. Notice how regularization stabilizes results.</li>
<li>Explore the cold-start problem: simulate a new user and try predicting preferences from minimal data.</li>
</ol>
<p>Low-rank models reveal a powerful truth: behind the enormous variety of human choices lies a surprisingly small set of underlying patterns-and linear algebra is the key to uncovering them.</p>
</section>
</section>
<section id="pagerank-and-random-walks-ranking-with-eigenvectors" class="level3">
<h3 class="anchored" data-anchor-id="pagerank-and-random-walks-ranking-with-eigenvectors">98. PageRank and Random Walks (Ranking with Eigenvectors)</h3>
<p>PageRank, the algorithm that once powered Google’s search engine dominance, is a striking example of how linear algebra and eigenvectors can measure importance in a network. At its core, it models the web as a graph and asks a simple question: if you randomly surf the web forever, which pages will you visit most often?</p>
<section id="the-web-as-a-graph" class="level4">
<h4 class="anchored" data-anchor-id="the-web-as-a-graph">The Web as a Graph</h4>
<ul>
<li>Each web page is a node.</li>
<li>Each hyperlink is a directed edge.</li>
<li>The adjacency matrix <span class="math inline">\(A\)</span> encodes which pages link to which:</li>
</ul>
<p><span class="math display">\[
A_{ij} = 1 \quad \text{if page \(j\) links to page \(i\)}.
\]</span></p>
<p>Why columns instead of rows? Because links flow from source to destination, and PageRank naturally arises when analyzing column-stochastic transition matrices.</p>
</section>
<section id="transition-matrix" class="level4">
<h4 class="anchored" data-anchor-id="transition-matrix">Transition Matrix</h4>
<p>To model random surfing, we define a column-stochastic matrix <span class="math inline">\(P\)</span>:</p>
<p><span class="math display">\[
P_{ij} = \frac{1}{\text{outdeg}(j)} \quad \text{if \(j \to i\)}.
\]</span></p>
<ul>
<li>Each column sums to 1.</li>
<li><span class="math inline">\(P_{ij}\)</span> is the probability of moving from page <span class="math inline">\(j\)</span> to page <span class="math inline">\(i\)</span>.</li>
<li>This defines a Markov chain: a random process where the next state depends only on the current one.</li>
</ul>
<p>If a user is on page <span class="math inline">\(j\)</span>, they pick one outgoing link uniformly at random.</p>
</section>
<section id="random-walk-interpretation" class="level4">
<h4 class="anchored" data-anchor-id="random-walk-interpretation">Random Walk Interpretation</h4>
<p>Imagine a web surfer moving page by page according to <span class="math inline">\(P\)</span>. After many steps, the fraction of time spent on each page converges to a steady-state distribution vector <span class="math inline">\(\pi\)</span>:</p>
<p><span class="math display">\[
\pi = P \pi.
\]</span></p>
<p>This is an eigenvector equation: <span class="math inline">\(\pi\)</span> is the stationary eigenvector of <span class="math inline">\(P\)</span> with eigenvalue 1.</p>
<ul>
<li><span class="math inline">\(\pi_i\)</span> is the long-run probability of being on page <span class="math inline">\(i\)</span>.</li>
<li>A higher <span class="math inline">\(\pi_i\)</span> means greater importance.</li>
</ul>
</section>
<section id="the-pagerank-adjustment-teleportation" class="level4">
<h4 class="anchored" data-anchor-id="the-pagerank-adjustment-teleportation">The PageRank Adjustment: Teleportation</h4>
<p>The pure random walk has problems:</p>
<ol type="1">
<li>Dead ends: Pages with no outgoing links trap the surfer.</li>
<li>Spider traps: Groups of pages linking only to each other hoard probability mass.</li>
</ol>
<p>Solution: add a teleportation mechanism:</p>
<ul>
<li>With probability <span class="math inline">\(\alpha\)</span> (say 0.85), follow a link.</li>
<li>With probability <span class="math inline">\(1-\alpha\)</span>, jump to a random page.</li>
</ul>
<p>This defines the PageRank matrix:</p>
<p><span class="math display">\[
M = \alpha P + (1-\alpha)\frac{1}{n} ee^T,
\]</span></p>
<p>where <span class="math inline">\(e\)</span> is the all-ones vector.</p>
<ul>
<li><span class="math inline">\(M\)</span> is stochastic, irreducible, and aperiodic.</li>
<li>By the Perron–Frobenius theorem, it has a unique stationary distribution <span class="math inline">\(\pi\)</span>.</li>
</ul>
</section>
<section id="solving-the-eigenproblem" class="level4">
<h4 class="anchored" data-anchor-id="solving-the-eigenproblem">Solving the Eigenproblem</h4>
<p>The PageRank vector <span class="math inline">\(\pi\)</span> satisfies:</p>
<p><span class="math display">\[
M \pi = \pi.
\]</span></p>
<ul>
<li>Computing <span class="math inline">\(\pi\)</span> directly via eigen-decomposition is infeasible for billions of pages.</li>
<li>Instead, use power iteration: repeatedly multiply a vector by <span class="math inline">\(M\)</span> until convergence.</li>
</ul>
<p>This works because the largest eigenvalue is 1, and the method converges to its eigenvector.</p>
</section>
<section id="worked-example-a-tiny-web" class="level4">
<h4 class="anchored" data-anchor-id="worked-example-a-tiny-web">Worked Example: A Tiny Web</h4>
<p>Suppose 3 pages with links:</p>
<ul>
<li>Page 1 → Page 2</li>
<li>Page 2 → Page 3</li>
<li>Page 3 → Page 1 and Page 2</li>
</ul>
<p>Adjacency matrix (columns = source):</p>
<p><span class="math display">\[
A = \begin{bmatrix}
0 &amp; 0 &amp; 1 \\
1 &amp; 0 &amp; 1 \\
0 &amp; 1 &amp; 0
\end{bmatrix}.
\]</span></p>
<p>Transition matrix:</p>
<p><span class="math display">\[
P = \begin{bmatrix}
0 &amp; 0 &amp; 1/2 \\
1 &amp; 0 &amp; 1/2 \\
0 &amp; 1 &amp; 0
\end{bmatrix}.
\]</span></p>
<p>With teleportation (<span class="math inline">\(\alpha=0.85\)</span>), we form <span class="math inline">\(M\)</span>. Power iteration quickly converges to <span class="math inline">\(\pi = [0.37, 0.34, 0.29]^T\)</span>. Page 1 is ranked highest.</p>
</section>
<section id="beyond-the-web" class="level4">
<h4 class="anchored" data-anchor-id="beyond-the-web">Beyond the Web</h4>
<p>Although born in search engines, PageRank’s mathematics applies broadly:</p>
<ul>
<li>Social networks: Rank influential users by their connections.</li>
<li>Citation networks: Rank scientific papers by how they are referenced.</li>
<li>Biology: Identify key proteins in protein–protein interaction networks.</li>
<li>Recommendation systems: Rank products or movies via link structures.</li>
</ul>
<p>In each case, importance is defined not by how many connections a node has, but by the importance of the nodes that point to it.</p>
</section>
<section id="computational-challenges" class="level4">
<h4 class="anchored" data-anchor-id="computational-challenges">Computational Challenges</h4>
<ul>
<li>Scale: Billions of pages mean <span class="math inline">\(M\)</span> cannot be stored fully; sparse matrix techniques are essential.</li>
<li>Convergence: Power iteration may take hundreds of steps; preconditioning and parallelization speed it up.</li>
<li>Personalization: Instead of uniform teleportation, adjust probabilities to bias toward user interests.</li>
</ul>
</section>
<section id="why-it-matters-93" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-93">Why It Matters</h4>
<p>PageRank illustrates a deep principle: importance emerges from connectivity. Linear algebra captures this by identifying the dominant eigenvector of a transition matrix. This idea-ranking nodes in a network by stationary distributions-has transformed search engines, social media, and science itself.</p>
</section>
<section id="try-it-yourself-97" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-97">Try It Yourself</h4>
<ol type="1">
<li>Construct a 4-page web graph and compute its PageRank manually with <span class="math inline">\(\alpha = 0.85\)</span>.</li>
<li>Implement power iteration in Python or MATLAB for a small adjacency matrix.</li>
<li>Compare PageRank to simple degree counts. Notice how PageRank rewards links from important nodes more heavily.</li>
<li>Modify teleportation to bias toward a subset of pages (personalized PageRank). Observe how rankings change.</li>
</ol>
<p>PageRank is not only a milestone in computer science history-it is a living example of how eigenvectors can capture global importance from local structure.</p>
</section>
</section>
<section id="numerical-linear-algebra-essentials-floating-point-blaslapack" class="level3">
<h3 class="anchored" data-anchor-id="numerical-linear-algebra-essentials-floating-point-blaslapack">99. Numerical Linear Algebra Essentials (Floating Point, BLAS/LAPACK)</h3>
<p>Linear algebra in theory is exact: numbers behave like real numbers, operations are deterministic, and results are precise. In practice, computations are carried out on computers, where numbers are represented in finite precision and algorithms must balance speed, accuracy, and stability. This intersection-numerical linear algebra-is what makes linear algebra usable at modern scales.</p>
<section id="floating-point-representation" class="level4">
<h4 class="anchored" data-anchor-id="floating-point-representation">Floating-Point Representation</h4>
<p>Real numbers cannot be stored exactly on a digital machine. Instead, they are approximated using the IEEE 754 floating-point standard.</p>
<ul>
<li><p>A floating-point number is stored as:</p>
<p><span class="math display">\[
x = \pm (1.m_1 m_2 m_3 \dots) \times 2^e
\]</span></p>
<p>where <span class="math inline">\(m\)</span> is the mantissa and <span class="math inline">\(e\)</span> is the exponent.</p></li>
<li><p>Single precision (float32): 32 bits → ~7 decimal digits of precision.</p></li>
<li><p>Double precision (float64): 64 bits → ~16 decimal digits.</p></li>
<li><p>Machine epsilon (<span class="math inline">\(\epsilon\)</span>): The smallest gap between 1 and the next representable number. For double precision, <span class="math inline">\(\epsilon \approx 2.22 \times 10^{-16}\)</span>.</p></li>
</ul>
<p>Implication: operations like subtraction of nearly equal numbers cause catastrophic cancellation, where significant digits vanish.</p>
</section>
<section id="conditioning-of-problems" class="level4">
<h4 class="anchored" data-anchor-id="conditioning-of-problems">Conditioning of Problems</h4>
<p>A linear algebra problem may be well-posed mathematically but still numerically difficult.</p>
<ul>
<li><p>The condition number of a matrix <span class="math inline">\(A\)</span>:</p>
<p><span class="math display">\[
\kappa(A) = \|A\| \cdot \|A^{-1}\|.
\]</span></p></li>
<li><p>If <span class="math inline">\(\kappa(A)\)</span> is large, small input errors cause large output errors.</p></li>
<li><p>Example: solving <span class="math inline">\(Ax = b\)</span>. With ill-conditioned <span class="math inline">\(A\)</span>, the computed solution may be unstable even with perfect algorithms.</p></li>
</ul>
<p>Geometric intuition: ill-conditioned matrices stretch vectors unevenly, so small perturbations in direction blow up under inversion.</p>
</section>
<section id="stability-of-algorithms" class="level4">
<h4 class="anchored" data-anchor-id="stability-of-algorithms">Stability of Algorithms</h4>
<ul>
<li>An algorithm is numerically stable if it controls the growth of errors from finite precision.</li>
<li>Gaussian elimination with partial pivoting is stable; without pivoting, it may fail catastrophically.</li>
<li>Orthogonal factorizations (QR, SVD) are usually more stable than elimination methods.</li>
</ul>
<p>Numerical analysis focuses on designing algorithms that guarantee accuracy within a few multiples of machine epsilon.</p>
</section>
<section id="direct-vs.-iterative-methods" class="level4">
<h4 class="anchored" data-anchor-id="direct-vs.-iterative-methods">Direct vs.&nbsp;Iterative Methods</h4>
<ol type="1">
<li><p>Direct methods: Solve in a finite number of steps (e.g., Gaussian elimination, LU decomposition, Cholesky for positive definite systems).</p>
<ul>
<li>Reliable for small/medium problems.</li>
<li>Complexity ~ <span class="math inline">\(O(n^3)\)</span>.</li>
</ul></li>
<li><p>Iterative methods: Generate successive approximations (e.g., Jacobi, Gauss–Seidel, Conjugate Gradient).</p>
<ul>
<li>Useful for very large, sparse systems.</li>
<li>Complexity per iteration ~ <span class="math inline">\(O(n^2)\)</span> or less, often leveraging sparsity.</li>
</ul></li>
</ol>
</section>
<section id="matrix-factorizations-in-computation" class="level4">
<h4 class="anchored" data-anchor-id="matrix-factorizations-in-computation">Matrix Factorizations in Computation</h4>
<p>Many algorithms rely on factorizing a matrix once, then reusing it:</p>
<ul>
<li>LU decomposition: Efficient for solving multiple right-hand sides.</li>
<li>QR factorization: Stable approach for least squares.</li>
<li>SVD: Gold standard for ill-conditioned problems, though expensive.</li>
</ul>
<p>These factorizations reduce repeated operations into structured, cache-friendly steps.</p>
</section>
<section id="sparse-vs.-dense-computations" class="level4">
<h4 class="anchored" data-anchor-id="sparse-vs.-dense-computations">Sparse vs.&nbsp;Dense Computations</h4>
<ul>
<li>Dense matrices: Most entries are nonzero. Use dense linear algebra packages like BLAS and LAPACK.</li>
<li>Sparse matrices: Most entries are zero. Store only nonzeros, use specialized algorithms to avoid wasted computation.</li>
</ul>
<p>Large-scale problems (e.g., finite element simulations, web graphs) are feasible only because of sparse methods.</p>
</section>
<section id="blas-and-lapack-standard-libraries" class="level4">
<h4 class="anchored" data-anchor-id="blas-and-lapack-standard-libraries">BLAS and LAPACK: Standard Libraries</h4>
<ul>
<li>BLAS (Basic Linear Algebra Subprograms): Defines kernels for vector and matrix operations (dot products, matrix–vector, matrix–matrix multiplication). Optimized BLAS implementations exploit cache, SIMD, and multi-core parallelism.</li>
<li>LAPACK (Linear Algebra PACKage): Builds on BLAS to provide algorithms for solving systems, eigenvalue problems, SVD, etc. LAPACK is the backbone of many scientific computing environments (MATLAB, NumPy, Julia).</li>
<li>MKL, OpenBLAS, cuBLAS: Vendor-specific implementations optimized for Intel CPUs, open-source systems, or NVIDIA GPUs.</li>
</ul>
<p>These libraries make the difference between code that runs in minutes and code that runs in milliseconds.</p>
</section>
<section id="floating-point-pitfalls" class="level4">
<h4 class="anchored" data-anchor-id="floating-point-pitfalls">Floating-Point Pitfalls</h4>
<ol type="1">
<li>Accumulated round-off: Summing numbers of vastly different magnitudes may discard small contributions.</li>
<li>Loss of orthogonality: Repeated Gram–Schmidt orthogonalization without reorthogonalization may drift numerically.</li>
<li>Overflow/underflow: Extremely large/small numbers exceed representable range.</li>
<li>NaNs and Infs: Divide-by-zero or invalid operations propagate errors.</li>
</ol>
<p>Mitigation: use numerically stable algorithms, scale inputs, and check condition numbers.</p>
</section>
<section id="parallel-and-gpu-computing" class="level4">
<h4 class="anchored" data-anchor-id="parallel-and-gpu-computing">Parallel and GPU Computing</h4>
<p>Modern numerical linear algebra thrives on parallelism:</p>
<ul>
<li>GPUs accelerate dense linear algebra with thousands of cores (cuBLAS, cuSOLVER).</li>
<li>Distributed libraries (ScaLAPACK, PETSc, Trilinos) allow solving problems with billions of unknowns across clusters.</li>
<li>Mixed precision methods: compute in float32 or even float16, then refine in float64, balancing speed and accuracy.</li>
</ul>
</section>
<section id="applications-in-the-real-world" class="level4">
<h4 class="anchored" data-anchor-id="applications-in-the-real-world">Applications in the Real World</h4>
<ul>
<li>Engineering simulations: Structural mechanics, fluid dynamics rely on sparse solvers.</li>
<li>Machine learning: Training deep networks depends on optimized BLAS for matrix multiplications.</li>
<li>Finance: Risk models solve huge regression problems with factorized covariance matrices.</li>
<li>Big data: Dimensionality reduction (PCA, SVD) requires large-scale, stable algorithms.</li>
</ul>
</section>
<section id="why-it-matters-94" class="level4">
<h4 class="anchored" data-anchor-id="why-it-matters-94">Why It Matters</h4>
<p>Linear algebra in practice is about more than theorems: it’s about turning abstract models into computations that run reliably on imperfect hardware. Numerical linear algebra provides the essential toolkit-floating-point understanding, conditioning analysis, stable algorithms, and optimized libraries-that ensures results are both fast and trustworthy.</p>
</section>
<section id="try-it-yourself-98" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-98">Try It Yourself</h4>
<ol type="1">
<li>Compute the condition number of a nearly singular matrix (e.g., <span class="math inline">\(\begin{bmatrix} 1 &amp; 1 \\ 1 &amp; 1.0001 \end{bmatrix}\)</span>) and solve <span class="math inline">\(Ax=b\)</span>. Compare results in single vs.&nbsp;double precision.</li>
<li>Implement Gaussian elimination with and without pivoting. Compare errors for ill-conditioned matrices.</li>
<li>Use NumPy with OpenBLAS to time large matrix multiplications; compare against a naive Python implementation.</li>
<li>Explore iterative solvers: implement Conjugate Gradient for a sparse symmetric positive definite system.</li>
</ol>
<p>Numerical linear algebra is the bridge between mathematical elegance and computational reality. It teaches us that solving equations on a computer is not just about the equations-it’s about the algorithms, representations, and hardware that bring them to life.</p>
</section>
</section>
<section id="capstone-problem-sets-and-next-steps-a-roadmap-to-mastery" class="level3">
<h3 class="anchored" data-anchor-id="capstone-problem-sets-and-next-steps-a-roadmap-to-mastery">100. Capstone Problem Sets and Next Steps (A Roadmap to Mastery)</h3>
<p>You’ve now walked through the major landmarks of linear algebra: vectors, matrices, systems, transformations, determinants, eigenvalues, orthogonality, SVD, and applications to data and networks. The journey doesn’t end here. This last section is designed as a capstone, a way to tie things together and show you how to keep practicing, exploring, and deepening your understanding. Think of it as your “next steps” map.</p>
<section id="practicing-the-basics-until-they-feel-natural" class="level4">
<h4 class="anchored" data-anchor-id="practicing-the-basics-until-they-feel-natural">Practicing the Basics Until They Feel Natural</h4>
<p>Linear algebra may seem heavy at first, but the simplest drills build lasting confidence. Try solving a few systems of equations by hand using elimination, and notice how pivoting reveals where solutions exist-or don’t. Write down a small matrix and practice multiplying it by a vector. This might feel mechanical, but it’s how your intuition sharpens: every time you push numbers through the rules, you’re learning how the algebra reshapes space.</p>
<p>Even a single concept, like the dot product, can teach a lot. Take two short vectors in the plane, compute their dot product, and then compare it to the cosine of the angle between them. Seeing algebra match geometry is what makes linear algebra come alive.</p>
</section>
<section id="moving-beyond-computation-understanding-structures" class="level4">
<h4 class="anchored" data-anchor-id="moving-beyond-computation-understanding-structures">Moving Beyond Computation: Understanding Structures</h4>
<p>Once you’re comfortable with the mechanics, try reflecting on the bigger structures. What does it mean for a set of vectors to be a subspace? Can you tell whether a line through the origin is one? What about a line shifted off the origin? This is where the rules and axioms you’ve seen start to guide your reasoning.</p>
<p>Experiment with bases and coordinates: pick two different bases for the plane and see how a single point looks different depending on the “ruler” you’re using. Write out the change-of-basis matrix and check that it transforms coordinates the way you expect. These exercises show that linear algebra isn’t just about numbers-it’s about perspective.</p>
</section>
<section id="bringing-ideas-together-in-larger-problems" class="level4">
<h4 class="anchored" data-anchor-id="bringing-ideas-together-in-larger-problems">Bringing Ideas Together in Larger Problems</h4>
<p>The real joy comes when different ideas collide. Suppose you have noisy data, like a scatter of points that should lie along a line. Try fitting a line using least squares. What you’re really doing is projecting the data onto a subspace. Or take a small Markov chain, like a random walk around three or four nodes, and compute its long-term distribution. That steady state is an eigenvector in disguise. These integrative problems demonstrate how the topics you’ve studied connect.</p>
<p>Projects make this even more vivid. For example:</p>
<ul>
<li>In computer graphics, write simple code that rotates or reflects a shape using a matrix.</li>
<li>In networks, use the Laplacian to identify clusters in a social graph of friends.</li>
<li>In recommendation systems, factorize a small user–item table to predict missing ratings.</li>
</ul>
<p>These aren’t abstract puzzles-they show how linear algebra works in the real world.</p>
</section>
<section id="looking-ahead-where-linear-algebra-leads-you" class="level4">
<h4 class="anchored" data-anchor-id="looking-ahead-where-linear-algebra-leads-you">Looking Ahead: Where Linear Algebra Leads You</h4>
<p>By now you know that linear algebra is not an isolated subject; it’s a foundation. The next steps depend on your interests.</p>
<p>If you enjoy computation, numerical linear algebra is the natural extension. It digs into how floating-point numbers behave on real machines, how to control round-off errors, and why some algorithms are more stable than others. You’ll learn why Gaussian elimination with pivoting is safe while without pivoting it can fail, and why QR and SVD are trusted in sensitive applications.</p>
<p>If abstraction intrigues you, then abstract linear algebra opens the door. Here you’ll move beyond <span class="math inline">\(\mathbb{R}^n\)</span> into general vector spaces: polynomials as vectors, functions as vectors, dual spaces, and eventually tensor products. These ideas power much of modern mathematics and physics.</p>
<p>If data excites you, statistics and machine learning are a natural path. Covariance matrices, principal component analysis, regression, and neural networks all rest on linear algebra. Understanding them deeply requires both the computation you’ve practiced and the geometric insights you’ve built.</p>
<p>And if your curiosity points toward the sciences, linear algebra is everywhere: in quantum mechanics, where states are vectors and operators are matrices; in engineering, where vibrations and control systems rely on eigenvalues; in computer graphics, where every rotation and projection is a linear transformation.</p>
</section>
<section id="why-this-capstone-matters" class="level4">
<h4 class="anchored" data-anchor-id="why-this-capstone-matters">Why This Capstone Matters</h4>
<p>This final step is less about new theorems and more about perspective. The problems you solve now-whether small drills or large projects-train you to see structure, not just numbers. The roadmap is open-ended, because linear algebra itself is open-ended: once you learn to see the world through its lens, you notice it everywhere, from the patterns in networks to the behavior of algorithms to the geometry of space.</p>
</section>
<section id="try-it-yourself-99" class="level4">
<h4 class="anchored" data-anchor-id="try-it-yourself-99">Try It Yourself</h4>
<ol type="1">
<li>Take a dataset you care about-maybe sports scores, songs you listen to, or spending records. Organize it as a matrix. Compute simple things: averages (centering), a regression line, maybe even principal components. See what structure you uncover.</li>
<li>Write a short program that solves systems of equations using elimination. Test it on well-behaved and nearly singular matrices. Notice how stability changes.</li>
<li>Draw a 2D scatterplot and fit a line with least squares. Plot the residuals. What does it mean geometrically that the residuals are orthogonal to the line?</li>
<li>Try explaining eigenvalues to a friend without formulas-just pictures and stories. Teaching it will make it real.</li>
</ol>
<p>Linear algebra is both a tool and a way of thinking. You now have enough to stand on your own, but the road continues forward-into deeper math, into practical computation, and into the sciences that rely on these ideas every day. This capstone is an invitation: keep practicing, keep exploring, and let the structures of linear algebra sharpen the way you see the world.</p>
</section>
<section id="closing-9" class="level4">
<h4 class="anchored" data-anchor-id="closing-9">Closing</h4>
<pre><code>From lines to the stars,
each problem bends, transforms, grows—
paths extend ahead.</code></pre>
</section>
</section>
<section id="finale" class="level3">
<h3 class="anchored" data-anchor-id="finale">Finale</h3>
<p><em>A quiet closing, where lessons settle and the music of algebra carries on beyond the final page.</em></p>
<p><strong>1. Quiet Reflection</strong></p>
<pre><code>Lessons intertwining,
the book rests, but vectors stretch—
silence holds their song.</code></pre>
<p><strong>2. Infinite Journey</strong></p>
<pre><code>One map now complete,
yet beyond each line and plane
new horizons call.</code></pre>
<p><strong>3. Structure and Growth</strong></p>
<pre><code>Roots beneath the ground,
branches weaving endless skies,
algebra takes flight.</code></pre>
<p><strong>4. Light After Study</strong></p>
<pre><code>Numbers fade to light,
patterns linger in the mind,
paths remain open.</code></pre>
<p><strong>5. Eternal Motion</strong></p>
<pre><code>Stillness finds its place,
transformations carry on,
movement without end.</code></pre>
<p><strong>6. Gratitude and Closure</strong></p>
<pre><code>Steps of thought complete,
spaces carved with gentle care,
thank you, wandering mind.</code></pre>
<p><strong>7. Future Echo</strong></p>
<pre><code>From shadows to form,
each question births new echoes—
the journey goes on.</code></pre>
<p><strong>8. Horizon Beyond</strong></p>
<pre><code>The book closes here,
yet the lines refuse to end,
they stretch toward the stars.</code></pre>


</section>
</section>

</main> <!-- /main -->
<script id="quarto-html-after-body" type="application/javascript">
  window.document.addEventListener("DOMContentLoaded", function (event) {
    const icon = "";
    const anchorJS = new window.AnchorJS();
    anchorJS.options = {
      placement: 'right',
      icon: icon
    };
    anchorJS.add('.anchored');
    const isCodeAnnotation = (el) => {
      for (const clz of el.classList) {
        if (clz.startsWith('code-annotation-')) {                     
          return true;
        }
      }
      return false;
    }
    const onCopySuccess = function(e) {
      // button target
      const button = e.trigger;
      // don't keep focus
      button.blur();
      // flash "checked"
      button.classList.add('code-copy-button-checked');
      var currentTitle = button.getAttribute("title");
      button.setAttribute("title", "Copied!");
      let tooltip;
      if (window.bootstrap) {
        button.setAttribute("data-bs-toggle", "tooltip");
        button.setAttribute("data-bs-placement", "left");
        button.setAttribute("data-bs-title", "Copied!");
        tooltip = new bootstrap.Tooltip(button, 
          { trigger: "manual", 
            customClass: "code-copy-button-tooltip",
            offset: [0, -8]});
        tooltip.show();    
      }
      setTimeout(function() {
        if (tooltip) {
          tooltip.hide();
          button.removeAttribute("data-bs-title");
          button.removeAttribute("data-bs-toggle");
          button.removeAttribute("data-bs-placement");
        }
        button.setAttribute("title", currentTitle);
        button.classList.remove('code-copy-button-checked');
      }, 1000);
      // clear code selection
      e.clearSelection();
    }
    const getTextToCopy = function(trigger) {
      const outerScaffold = trigger.parentElement.cloneNode(true);
      const codeEl = outerScaffold.querySelector('code');
      for (const childEl of codeEl.children) {
        if (isCodeAnnotation(childEl)) {
          childEl.remove();
        }
      }
      return codeEl.innerText;
    }
    const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
      text: getTextToCopy
    });
    clipboard.on('success', onCopySuccess);
    if (window.document.getElementById('quarto-embedded-source-code-modal')) {
      const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
        text: getTextToCopy,
        container: window.document.getElementById('quarto-embedded-source-code-modal')
      });
      clipboardModal.on('success', onCopySuccess);
    }
      var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
      var mailtoRegex = new RegExp(/^mailto:/);
        var filterRegex = new RegExp('/' + window.location.host + '/');
      var isInternal = (href) => {
          return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
      }
      // Inspect non-navigation links and adorn them if external
     var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
      for (var i=0; i<links.length; i++) {
        const link = links[i];
        if (!isInternal(link.href)) {
          // undo the damage that might have been done by quarto-nav.js in the case of
          // links that we want to consider external
          if (link.dataset.originalHref !== undefined) {
            link.href = link.dataset.originalHref;
          }
        }
      }
    function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
      const config = {
        allowHTML: true,
        maxWidth: 500,
        delay: 100,
        arrow: false,
        appendTo: function(el) {
            return el.parentElement;
        },
        interactive: true,
        interactiveBorder: 10,
        theme: 'quarto',
        placement: 'bottom-start',
      };
      if (contentFn) {
        config.content = contentFn;
      }
      if (onTriggerFn) {
        config.onTrigger = onTriggerFn;
      }
      if (onUntriggerFn) {
        config.onUntrigger = onUntriggerFn;
      }
      window.tippy(el, config); 
    }
    const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
    for (var i=0; i<noterefs.length; i++) {
      const ref = noterefs[i];
      tippyHover(ref, function() {
        // use id or data attribute instead here
        let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
        try { href = new URL(href).hash; } catch {}
        const id = href.replace(/^#\/?/, "");
        const note = window.document.getElementById(id);
        if (note) {
          return note.innerHTML;
        } else {
          return "";
        }
      });
    }
    const xrefs = window.document.querySelectorAll('a.quarto-xref');
    const processXRef = (id, note) => {
      // Strip column container classes
      const stripColumnClz = (el) => {
        el.classList.remove("page-full", "page-columns");
        if (el.children) {
          for (const child of el.children) {
            stripColumnClz(child);
          }
        }
      }
      stripColumnClz(note)
      if (id === null || id.startsWith('sec-')) {
        // Special case sections, only their first couple elements
        const container = document.createElement("div");
        if (note.children && note.children.length > 2) {
          container.appendChild(note.children[0].cloneNode(true));
          for (let i = 1; i < note.children.length; i++) {
            const child = note.children[i];
            if (child.tagName === "P" && child.innerText === "") {
              continue;
            } else {
              container.appendChild(child.cloneNode(true));
              break;
            }
          }
          if (window.Quarto?.typesetMath) {
            window.Quarto.typesetMath(container);
          }
          return container.innerHTML
        } else {
          if (window.Quarto?.typesetMath) {
            window.Quarto.typesetMath(note);
          }
          return note.innerHTML;
        }
      } else {
        // Remove any anchor links if they are present
        const anchorLink = note.querySelector('a.anchorjs-link');
        if (anchorLink) {
          anchorLink.remove();
        }
        if (window.Quarto?.typesetMath) {
          window.Quarto.typesetMath(note);
        }
        if (note.classList.contains("callout")) {
          return note.outerHTML;
        } else {
          return note.innerHTML;
        }
      }
    }
    for (var i=0; i<xrefs.length; i++) {
      const xref = xrefs[i];
      tippyHover(xref, undefined, function(instance) {
        instance.disable();
        let url = xref.getAttribute('href');
        let hash = undefined; 
        if (url.startsWith('#')) {
          hash = url;
        } else {
          try { hash = new URL(url).hash; } catch {}
        }
        if (hash) {
          const id = hash.replace(/^#\/?/, "");
          const note = window.document.getElementById(id);
          if (note !== null) {
            try {
              const html = processXRef(id, note.cloneNode(true));
              instance.setContent(html);
            } finally {
              instance.enable();
              instance.show();
            }
          } else {
            // See if we can fetch this
            fetch(url.split('#')[0])
            .then(res => res.text())
            .then(html => {
              const parser = new DOMParser();
              const htmlDoc = parser.parseFromString(html, "text/html");
              const note = htmlDoc.getElementById(id);
              if (note !== null) {
                const html = processXRef(id, note);
                instance.setContent(html);
              } 
            }).finally(() => {
              instance.enable();
              instance.show();
            });
          }
        } else {
          // See if we can fetch a full url (with no hash to target)
          // This is a special case and we should probably do some content thinning / targeting
          fetch(url)
          .then(res => res.text())
          .then(html => {
            const parser = new DOMParser();
            const htmlDoc = parser.parseFromString(html, "text/html");
            const note = htmlDoc.querySelector('main.content');
            if (note !== null) {
              // This should only happen for chapter cross references
              // (since there is no id in the URL)
              // remove the first header
              if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
                note.children[0].remove();
              }
              const html = processXRef(null, note);
              instance.setContent(html);
            } 
          }).finally(() => {
            instance.enable();
            instance.show();
          });
        }
      }, function(instance) {
      });
    }
        let selectedAnnoteEl;
        const selectorForAnnotation = ( cell, annotation) => {
          let cellAttr = 'data-code-cell="' + cell + '"';
          let lineAttr = 'data-code-annotation="' +  annotation + '"';
          const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
          return selector;
        }
        const selectCodeLines = (annoteEl) => {
          const doc = window.document;
          const targetCell = annoteEl.getAttribute("data-target-cell");
          const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
          const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
          const lines = annoteSpan.getAttribute("data-code-lines").split(",");
          const lineIds = lines.map((line) => {
            return targetCell + "-" + line;
          })
          let top = null;
          let height = null;
          let parent = null;
          if (lineIds.length > 0) {
              //compute the position of the single el (top and bottom and make a div)
              const el = window.document.getElementById(lineIds[0]);
              top = el.offsetTop;
              height = el.offsetHeight;
              parent = el.parentElement.parentElement;
            if (lineIds.length > 1) {
              const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
              const bottom = lastEl.offsetTop + lastEl.offsetHeight;
              height = bottom - top;
            }
            if (top !== null && height !== null && parent !== null) {
              // cook up a div (if necessary) and position it 
              let div = window.document.getElementById("code-annotation-line-highlight");
              if (div === null) {
                div = window.document.createElement("div");
                div.setAttribute("id", "code-annotation-line-highlight");
                div.style.position = 'absolute';
                parent.appendChild(div);
              }
              div.style.top = top - 2 + "px";
              div.style.height = height + 4 + "px";
              div.style.left = 0;
              let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
              if (gutterDiv === null) {
                gutterDiv = window.document.createElement("div");
                gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
                gutterDiv.style.position = 'absolute';
                const codeCell = window.document.getElementById(targetCell);
                const gutter = codeCell.querySelector('.code-annotation-gutter');
                gutter.appendChild(gutterDiv);
              }
              gutterDiv.style.top = top - 2 + "px";
              gutterDiv.style.height = height + 4 + "px";
            }
            selectedAnnoteEl = annoteEl;
          }
        };
        const unselectCodeLines = () => {
          const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
          elementsIds.forEach((elId) => {
            const div = window.document.getElementById(elId);
            if (div) {
              div.remove();
            }
          });
          selectedAnnoteEl = undefined;
        };
          // Handle positioning of the toggle
      window.addEventListener(
        "resize",
        throttle(() => {
          elRect = undefined;
          if (selectedAnnoteEl) {
            selectCodeLines(selectedAnnoteEl);
          }
        }, 10)
      );
      function throttle(fn, ms) {
      let throttle = false;
      let timer;
        return (...args) => {
          if(!throttle) { // first call gets through
              fn.apply(this, args);
              throttle = true;
          } else { // all the others get throttled
              if(timer) clearTimeout(timer); // cancel #2
              timer = setTimeout(() => {
                fn.apply(this, args);
                timer = throttle = false;
              }, ms);
          }
        };
      }
        // Attach click handler to the DT
        const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
        for (const annoteDlNode of annoteDls) {
          annoteDlNode.addEventListener('click', (event) => {
            const clickedEl = event.target;
            if (clickedEl !== selectedAnnoteEl) {
              unselectCodeLines();
              const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
              if (activeEl) {
                activeEl.classList.remove('code-annotation-active');
              }
              selectCodeLines(clickedEl);
              clickedEl.classList.add('code-annotation-active');
            } else {
              // Unselect the line
              unselectCodeLines();
              clickedEl.classList.remove('code-annotation-active');
            }
          });
        }
    const findCites = (el) => {
      const parentEl = el.parentElement;
      if (parentEl) {
        const cites = parentEl.dataset.cites;
        if (cites) {
          return {
            el,
            cites: cites.split(' ')
          };
        } else {
          return findCites(el.parentElement)
        }
      } else {
        return undefined;
      }
    };
    var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
    for (var i=0; i<bibliorefs.length; i++) {
      const ref = bibliorefs[i];
      const citeInfo = findCites(ref);
      if (citeInfo) {
        tippyHover(citeInfo.el, function() {
          var popup = window.document.createElement('div');
          citeInfo.cites.forEach(function(cite) {
            var citeDiv = window.document.createElement('div');
            citeDiv.classList.add('hanging-indent');
            citeDiv.classList.add('csl-entry');
            var biblioDiv = window.document.getElementById('ref-' + cite);
            if (biblioDiv) {
              citeDiv.innerHTML = biblioDiv.innerHTML;
            }
            popup.appendChild(citeDiv);
          });
          return popup.innerHTML;
        });
      }
    }
  });
  </script>
<nav class="page-navigation">
  <div class="nav-page nav-page-previous">
      <a href="../../index.html" class="pagination-link" aria-label="Content">
        <i class="bi bi-arrow-left-short"></i> <span class="nav-page-text"><span class="chapter-title">Content</span></span>
      </a>          
  </div>
  <div class="nav-page nav-page-next">
      <a href="../../books/en-US/lab.html" class="pagination-link" aria-label="The LAB">
        <span class="nav-page-text"><span class="chapter-title">The LAB</span></span> <i class="bi bi-arrow-right-short"></i>
      </a>
  </div>
</nav>
</div> <!-- /content -->




</body></html>